{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aa460049-640b-4961-981a-affc463c8a3b",
   "metadata": {},
   "source": [
    "# 1 准备工作"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3956f857-03ce-481c-bb42-abbe139311bd",
   "metadata": {},
   "source": [
    "## 1.1 安装相关包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3fb17e0f-b216-45a1-874a-89b4aedbd88e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !apt update\n",
    "# !apt install unrar -y\n",
    "# !apt install mdbtools -y\n",
    "# !apt install default-jre -y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "acee0a2a-18b5-463a-adb4-d636acf3ea62",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !java --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bb4535f7-9daf-474e-86a0-301054e782c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install pytz\n",
    "# !pip install rarfile\n",
    "# !pip install jaydebeapi\n",
    "# !pip install pyodbc\n",
    "# !pip install uuid\n",
    "# !pip install tenacity"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78abd657-675f-4d8c-8251-52752fef53b5",
   "metadata": {},
   "source": [
    "## 1.2 加载包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3a0df1bd-91b1-476d-bf6a-9c17983db382",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pytz\n",
    "from datetime import datetime\n",
    "import json, re, gc\n",
    "import glob\n",
    "import os, sys, rarfile\n",
    "from tqdm import tqdm\n",
    "# from typing import Dict, Any\n",
    "from pathlib import Path\n",
    "\n",
    "import platform\n",
    "import jaydebeapi\n",
    "import jpype\n",
    "import jpype.imports\n",
    "from jpype.types import *\n",
    "import subprocess\n",
    "import pyodbc, io\n",
    "import difflib\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "import torch\n",
    "import pickle\n",
    "# from openai import OpenAI\n",
    "import warnings, logging\n",
    "sys.path.append('./')\n",
    "\n",
    "from utils.load_access_data import *\n",
    "from utils.file_unrar import *\n",
    "# from utils.data_mapping import *\n",
    "# from utils.data_mapping_36country3 import *\n",
    "# from utils.invoke_llm import *\n",
    "from utils.file_process import *\n",
    "from utils.bash_mapping import *\n",
    "from utils.logging_config import setup_logger\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "logger = setup_logger('Data Goverance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0d22cb7f-a935-4e82-bb54-a1cc22e6d6a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1e70e112-297d-41ad-b94f-ef1c9bb884e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-17 07:05:13,601 - Data Goverance - INFO - 2025-09-17 15:05:13.600928+08:00\n"
     ]
    }
   ],
   "source": [
    "# 计时\n",
    "tz_beijing = pytz.timezone('Asia/Shanghai')\n",
    "bj_now = datetime.now(tz_beijing)\n",
    "dt_now = datetime.strptime(datetime.strftime(bj_now, '%Y-%m-%d'), '%Y-%m-%d')\n",
    "logger.info(bj_now)\n",
    "\n",
    "all_start=time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14176eb0-f67a-4053-96f8-ff299df631ae",
   "metadata": {},
   "source": [
    "## 1.3 准备全局变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2baa7fe2-b85d-4299-b235-d672e338ab78",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config:\n",
    "    target_cols = ['channelno', 'dataid', 'iesign', 'datatype', 'writeoffflag'\n",
    "            , 'writeoffdataid', 'outputdate', 'origincountrycode', 'origincountry', 'countrycodeofdelivery'\n",
    "            , 'countryofdelivery', 'importername', 'importeraddress', 'importercontact', 'suppliername'\n",
    "            , 'supplieraddress', 'suppliercontact', 'hscode', 'hscodedescription', 'commoditydescription'\n",
    "            , 'totalcifvalue', 'totalfobvalue', 'grossweight', 'netweight', 'quantity'\n",
    "            , 'quantityunit', 'teu', 'importer_forwarderagent', 'supplier_forwarderagent', 'abnormaldata'\n",
    "            , 'portofloading', 'portofdestination', 'loadingcountrycode', 'loadingcountry', 'transportterm'\n",
    "            , 'tradeterm', 'paymentterm', 'carrier', 'containerno', 'vesselname'\n",
    "            , 'brand', 'version', 'country', 'IMPORTER_ID', 'SUPPLIER_ID', 'cif_currency', 'fob_currency']\n",
    "config = Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "90881866-9cbb-4442-a5f6-7217e901ce3a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 加载国家映射表\n",
    "country_code = pd.read_csv('param_files/国家代码表.csv')\n",
    "country_code_dict = country_code.loc[:,['英文国名','中文国名']].to_dict('records')\n",
    "country_code_dict_detail = country_code.to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "67fbe1a5-8a7b-4d95-ae58-ef5a97913cc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载36个国家码\n",
    "data_country36_code = pd.read_csv('param_files/数据36国家代码表.csv')\n",
    "data_country36_dict = data_country36_code.to_dict('records')\n",
    "\n",
    "country36_dict = dict()\n",
    "for x in data_country36_dict:\n",
    "    country36_dict[x.get('中文国名')]={'英文国名': x.get('英文国名'),'国别编码':x.get('国别编码')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ff0099c1-7091-40f2-8fe0-00618ab32c97",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理的中间结果，\n",
    "model_result = pickle.load(open('param_files/model_rslt_05_20250810.pickle', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0158c76f-e0df-44d3-9235-12945187e89b",
   "metadata": {},
   "outputs": [],
   "source": [
    "country36_cols_dict = model_result['all_country_cols_dict']\n",
    "all_country_cols_dict = model_result['all_country_cols_dict']\n",
    "\n",
    "all_country_list = model_result['all_country_list']\n",
    "llm_country_describe = model_result['llm_country_describe']\n",
    "llm_country_mapping = model_result['llm_country_mapping']\n",
    "country_mapping = model_result['country_mapping']\n",
    "currency_df = model_result['currency_df']\n",
    "currency_on_usd = model_result['currency_on_usd']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "919d9b79-5ccd-4bc7-a1f6-704526b90632",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['all_country_cols_dict', 'all_country_list', 'llm_country_describe', 'llm_country_mapping', 'country_mapping', 'currency_df', 'currency_on_usd'])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_result.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eb6ed96d-7eef-45e7-acbb-4ffeeb3760ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(model_result['country_mapping'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf6830bc-d070-4f45-a9de-13fceee9f39b",
   "metadata": {},
   "source": [
    "# 2 原始文件批量解压\n",
    "\n",
    "原始数据保存路径： dataset/data_20250724/2025年1-3月汇总\n",
    "\n",
    "批量解压至： dataset/data_20250724/20251t3_unrar”"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4fe99ca6-b41b-40d2-981c-880874d3acee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:47:47,397 - Data Goverance - INFO - number of rar_files:142, number of non_rar_files:7\n"
     ]
    }
   ],
   "source": [
    "# 定义要遍历的根目录\n",
    "root_dir = Path('dataset/data_20250724/2025年4-6月汇总')\n",
    "dir_paths, file_paths = get_dir_files(root_dir)\n",
    "\n",
    "# 获取需要解压的文件目录\n",
    "rar_file_paths = [x for x in file_paths if str(x).split('.')[-1]=='rar']\n",
    "nrar_file_paths = [x for x in file_paths if str(x).split('.')[-1]!='rar']\n",
    "logger.info(f'number of rar_files:{len(rar_file_paths)}, number of non_rar_files:{len(nrar_file_paths)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "90e89919-162e-4eda-b93f-e968e47a71d5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # 批量解压至“dataset/data_20250724/20254t6_unrar”\n",
    "# # 请谨慎执行，如果解压过请不要再运行\n",
    "# extract_rar_batch(file_paths, source_str='2025年4-6月汇总', replace_str='20254t6_unrar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e868c9d1-a325-4ffc-a0c7-baabdc443541",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # 检查 检查\n",
    "# !apt update && apt install rar unrar\n",
    "# !unrar x dataset/data_20250724/20254t6_unrar/乌克兰/UA_IMPORT_202506_MDB.rar dataset/data_20250724/20254t6_unrar/乌克兰\n",
    "# !unrar x dataset/data_20250724/20254t6_unrar/乌克兰/UA_EXPORT_202506_MDB.rar dataset/data_20250724/20254t6_unrar/乌克兰\n",
    "# !rm -rf dataset/data_20250724/20254t6_unrar/乌克兰/UA_IMPORT_202506_MDB.rar\n",
    "# !rm -rf dataset/data_20250724/20254t6_unrar/乌克兰/UA_EXPORT_202506_MDB.rar\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ca558fd-2b08-4340-847b-5c0cf25c3af4",
   "metadata": {},
   "source": [
    "# 3 解压的各种格式数据统一转为csv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20fdd3cb-ca1a-4c3e-af5e-de00d0aaa8f6",
   "metadata": {},
   "source": [
    "\n",
    "添加数据主键 dataid\n",
    "添加数据主键 country\n",
    "添加数据主键 iesign\n",
    "\n",
    "转换后数据路径为：dataset/data_20250724/20251t3_src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c80de384-6c66-4822-bbc9-fd04180d08c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "country_list = ['智利']\n",
    "\n",
    "# 获取文件目录，区分出口、进口数据\n",
    "from_dir = Path('dataset/data_20250724/20254t6_unrar')\n",
    "to_dir = Path('dataset/data_20250724/20254t6_src')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "805beda6-f192-4982-8690-72e74c96cde3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # 慎重跑这个结果，会全量抽取原始数据,\n",
    "# # *****dataid会重新生成*****\n",
    "# all_country_cols_dict = get_source_csv(country_list, from_dir, to_dir, country36_dict)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d67e4eee-67cf-4fa3-9104-02fd19ad8d6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ls -lth dataset/data_20250724/20254t6_src2/智利/export_paths\n",
    "# !ls -lth dataset/data_20250724/20254t6_src2/智利/import_paths\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bb1f0647-cd60-40d8-8078-21bab17646cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !mv dataset/data_20250724/20254t6_src2/智利/export_paths/202506-EXP.csv dataset/data_20250724/20254t6_src/智利/export_paths/\n",
    "# !mv dataset/data_20250724/20254t6_src2/智利/import_paths/202506-IMP.csv dataset/data_20250724/20254t6_src/智利/import_paths/\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "51608bd7-a923-42af-9712-ff60e8f69886",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ls -lth dataset/data_20250724/20254t6_src/智利/export_paths/\n",
    "# !ls -lth dataset/data_20250724/20254t6_src/智利/import_paths/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f5e2b67-8d9e-4f8f-b748-dfc18a4165c6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8000393-dcda-4f0a-88a1-f08931c028d5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "96353e48-3495-4199-ac9d-59939b21ed43",
   "metadata": {},
   "source": [
    "# 4 更正数据月份至version字段"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2e90119f-c8fd-4702-a6ea-d72bfae0f723",
   "metadata": {},
   "outputs": [],
   "source": [
    "root_dir = Path('dataset/data_20250724/20254t6_src')\n",
    "dir_paths, file_paths = get_dir_files(root_dir)\n",
    "file_tree = distinct_ie_data(root_dir, country_list)\n",
    "\n",
    "save_dir = Path('dataset/data_20250724/20254t6_src_version')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5882d3db-5198-4d42-8815-ec544887a908",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# country_list = ['智利']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0a30c055-d35e-4ab2-85d6-c722bfd51e49",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% 0/3 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:47:47,442 - File Process - INFO - 智利\n",
      "2025-09-10 06:47:47,442 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/智利/import_paths/202504-IMP.csv\n",
      "202504\n",
      "2025-09-10 06:47:53,761 - File Process - INFO - data: (390891, 47)\n",
      "2025-09-10 06:47:53,761 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/智利/import_paths/202504-IMP.csv\n",
      "2025-09-10 06:47:53,761 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/智利/import_paths/202505-IMP.csv\n",
      "202505\n",
      "2025-09-10 06:48:00,358 - File Process - INFO - data: (404234, 47)\n",
      "2025-09-10 06:48:00,360 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/智利/import_paths/202505-IMP.csv\n",
      "2025-09-10 06:48:00,361 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/智利/import_paths/202506-IMP.csv\n",
      "202506\n",
      "2025-09-10 06:48:06,858 - File Process - INFO - data: (400856, 47)\n",
      "2025-09-10 06:48:06,859 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/智利/import_paths/202506-IMP.csv\n",
      "2025-09-10 06:48:06,861 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/智利/export_paths/202506-EXP.csv\n",
      "202506\n",
      "2025-09-10 06:48:08,217 - File Process - INFO - data: (87003, 46)\n",
      "2025-09-10 06:48:08,219 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/智利/export_paths/202506-EXP.csv\n",
      "2025-09-10 06:48:08,221 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/智利/export_paths/202505-EXP.csv\n",
      "202505\n",
      "2025-09-10 06:48:09,590 - File Process - INFO - data: (91874, 46)\n",
      "2025-09-10 06:48:09,592 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/智利/export_paths/202505-EXP.csv\n",
      "2025-09-10 06:48:09,594 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/智利/export_paths/202504-EXP.csv\n",
      "202504\n",
      "2025-09-10 06:48:11,180 - File Process - INFO - data: (112769, 46)\n",
      "2025-09-10 06:48:11,181 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/智利/export_paths/202504-EXP.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33% 1/3 [00:23<00:47, 23.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:48:11,181 - File Process - INFO - 坦桑尼亚\n",
      "2025-09-10 06:48:11,182 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/坦桑尼亚/import_paths/202504-IMP-RAW.csv\n",
      "202504\n",
      "2025-09-10 06:48:19,562 - File Process - INFO - data: (387273, 74)\n",
      "2025-09-10 06:48:19,564 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/坦桑尼亚/import_paths/202504-IMP-RAW.csv\n",
      "2025-09-10 06:48:19,565 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/坦桑尼亚/import_paths/202506-IMP-RAW.csv\n",
      "202506\n",
      "2025-09-10 06:48:27,266 - File Process - INFO - data: (344016, 74)\n",
      "2025-09-10 06:48:27,268 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/坦桑尼亚/import_paths/202506-IMP-RAW.csv\n",
      "2025-09-10 06:48:27,270 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/坦桑尼亚/import_paths/202505-IMP-RAW.csv\n",
      "202505\n",
      "2025-09-10 06:48:33,963 - File Process - INFO - data: (310571, 74)\n",
      "2025-09-10 06:48:33,964 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/坦桑尼亚/import_paths/202505-IMP-RAW.csv\n",
      "2025-09-10 06:48:33,965 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/坦桑尼亚/export_paths/202504-TRANS-RAW.csv\n",
      "202504\n",
      "2025-09-10 06:48:36,734 - File Process - INFO - data: (128934, 74)\n",
      "2025-09-10 06:48:36,736 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:48:36,737 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/坦桑尼亚/export_paths/202504-EXP-RAW.csv\n",
      "202504\n",
      "2025-09-10 06:48:36,939 - File Process - INFO - data: (6534, 74)\n",
      "2025-09-10 06:48:36,939 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202504-EXP-RAW.csv\n",
      "2025-09-10 06:48:36,940 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/坦桑尼亚/export_paths/202505-EXP-RAW.csv\n",
      "202505\n",
      "2025-09-10 06:48:37,047 - File Process - INFO - data: (5331, 74)\n",
      "2025-09-10 06:48:37,047 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202505-EXP-RAW.csv\n",
      "2025-09-10 06:48:37,047 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/坦桑尼亚/export_paths/202506-TRANS-RAW.csv\n",
      "202506\n",
      "2025-09-10 06:48:39,921 - File Process - INFO - data: (138793, 74)\n",
      "2025-09-10 06:48:39,922 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:48:39,922 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/坦桑尼亚/export_paths/202506-EXP-RAW.csv\n",
      "202506\n",
      "2025-09-10 06:48:40,054 - File Process - INFO - data: (5108, 74)\n",
      "2025-09-10 06:48:40,055 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202506-EXP-RAW.csv\n",
      "2025-09-10 06:48:40,055 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/坦桑尼亚/export_paths/202505-TRANS-RAW.csv\n",
      "202505\n",
      "2025-09-10 06:48:42,629 - File Process - INFO - data: (125615, 74)\n",
      "2025-09-10 06:48:42,631 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202505-TRANS-RAW.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67% 2/3 [00:55<00:28, 28.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:48:42,634 - File Process - INFO - 乌克兰\n",
      "2025-09-10 06:48:42,636 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/乌克兰/import_paths/UA_IMPORT_202504.csv\n",
      "202504\n",
      "2025-09-10 06:48:57,714 - File Process - INFO - data: (739143, 33)\n",
      "2025-09-10 06:48:57,717 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/乌克兰/import_paths/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:48:57,719 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/乌克兰/import_paths/UA_IMPORT_202506.csv\n",
      "202506\n",
      "2025-09-10 06:49:12,676 - File Process - INFO - data: (744475, 33)\n",
      "2025-09-10 06:49:12,678 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/乌克兰/import_paths/UA_IMPORT_202506.csv\n",
      "2025-09-10 06:49:12,679 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/乌克兰/import_paths/UA_IMPORT_202505.csv\n",
      "202505\n",
      "2025-09-10 06:49:28,456 - File Process - INFO - data: (755738, 33)\n",
      "2025-09-10 06:49:28,456 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/乌克兰/import_paths/UA_IMPORT_202505.csv\n",
      "2025-09-10 06:49:28,456 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/乌克兰/export_paths/UA_EXPORT_202506.csv\n",
      "202506\n",
      "2025-09-10 06:49:31,651 - File Process - INFO - data: (134581, 33)\n",
      "2025-09-10 06:49:31,652 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/乌克兰/export_paths/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:49:31,653 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/乌克兰/export_paths/UA_EXPORT_202504.csv\n",
      "202504\n",
      "2025-09-10 06:49:34,799 - File Process - INFO - data: (135921, 33)\n",
      "2025-09-10 06:49:34,801 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/乌克兰/export_paths/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:49:34,802 - File Process - INFO - read data: dataset/data_20250724/20254t6_src/乌克兰/export_paths/UA_EXPORT_202505.csv\n",
      "202505\n",
      "2025-09-10 06:49:38,145 - File Process - INFO - data: (147189, 33)\n",
      "2025-09-10 06:49:38,146 - File Process - INFO - new data save: dataset/data_20250724/20254t6_src_version/乌克兰/export_paths/UA_EXPORT_202505.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% 3/3 [01:50<00:00, 36.90s/it]\n"
     ]
    }
   ],
   "source": [
    "get_data_month2(root_dir, save_dir, country_list, country36_cols_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d25bd35-e78d-4277-a748-19468983a494",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "878b0cc6-58ac-4ba3-a3b0-85949f5e4b6d",
   "metadata": {},
   "source": [
    "# 5 映射函数实现"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4530484a-9a00-471f-a938-f4533cbdd9c6",
   "metadata": {},
   "source": [
    "## 5.2 出口、进口批量跑映射函数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f11e091-6559-4b9f-a0f0-6c97d75c3f20",
   "metadata": {},
   "source": [
    "### 5.2.1 出口跑批"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bfcbf49a-bf8e-4945-b23b-d0d5647b94c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取文件目录，区分出口、进口数据\n",
    "root_dir = Path('dataset/data_20250724/20254t6_src_version')\n",
    "\n",
    "# csv结果文件保存路径\n",
    "csv_files_path = Path('dataset/data_20250724/20254t6_csv_save')\n",
    "\n",
    "country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# country_list = ['智利']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6a365e16-9c5b-4938-9729-bf4a71422aa6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:49:38,225 - Bash Mapping - INFO - 乌克兰\n",
      "2025-09-10 06:49:38,225 - Bash Mapping - INFO - 智利\n",
      "2025-09-10 06:49:38,226 - Bash Mapping - INFO - 坦桑尼亚\n",
      "2025-09-10 06:49:38,226 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/乌克兰/export_paths/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:49:38,226 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/智利/export_paths/202506-EXP.csv\n",
      "2025-09-10 06:49:38,226 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:49:39,028 - Data Mapping - INFO - imp_df columns: ['DAY' 'MONTH' 'YEAR' 'CONTROL_ID' 'CUSTOMS' 'EXPORTER_ID' 'VERIFIER'\n",
      " 'EXPORTER' 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT' 'VARIETEY' 'BRAND' 'DESC'\n",
      " 'DEST_COUNTRY' 'TRANS_TYPE' 'TRANS_CORP' 'SHIP_NAME' 'TYPE_OF_LOAD'\n",
      " 'ORIGIN_PORT' 'DEST_PORT' 'G_WEIGHT' 'QUANTITY' 'UNIT_OF_QUANTITY' 'FOB'\n",
      " 'FREIGHT' 'INSURANCE' 'CIF' 'FOB_UNIT' 'TYPE_OF_PACKAGE'\n",
      " 'EXPORTER_REGION' 'PACKAGES' 'PACKAGES_DESC' 'TRANS_CORP_COUNTRY'\n",
      " 'SALE_CONDITION' 'ECONOMIC_ZONE' 'EXPORTER_ECONOMIC_KEY'\n",
      " 'TRANS_DOC_NUMBER' 'TRANS_DOC_DATE' 'VOYAGE_NUMBER' 'INCOTERMS' 'PAYMENT'\n",
      " 'dataid' 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:49:39,348 - Data Mapping - INFO - Export data mapping and cleaning takes 0.02 minutes\n",
      "2025-09-10 06:49:39,413 - Bash Mapping - INFO - target_tab shape: (87003, 47)\n",
      "2025-09-10 06:49:40,181 - Data Mapping - INFO - imp_df columns: ['CUSTOMS_OFFICE_CODE' 'TANSAD_YY' 'TANSAD_DATE' 'TANSAD_SERIAL_NO'\n",
      " 'TANSAD_FULL_NO' 'REGIME_NAME' 'TANSAD_YYYYMM' 'RELEASE_YYYYMM'\n",
      " 'PROCESSING_STATUS_NAME' 'ORIGIN_COUNTRY' 'CONSIGNMENT_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'EXPORT_COUNTRY' 'DEST_COUNTRY' 'CRN'\n",
      " 'MANIFEST_SUBMISSION_DATE' 'BL_NO' 'IS_CONTAINERIZED' 'NO_OF_CONTAINERS'\n",
      " 'VESSEL_NAME' 'TRANSPORT_MEANS_ID' 'TRANSPORT_NATIONALITY_CD'\n",
      " 'ARRIVAL_DATE' 'DISCHARGE_DATE' 'ENTRY_OFFICE_CODE'\n",
      " 'SELECTIVITY_RESULT_CD' 'CLEARANCE_PLAN_CD' 'TRANSPORT_MODE_CODE'\n",
      " 'TRANSPORT_MODE_NAME' 'IMPORTER_TAX_ID' 'IMPORTER_NAME' 'EXPORTER_TAX_ID'\n",
      " 'EXPORTER_NAME' 'DECLARANT_TAX_ID' 'DECLARANT_NAME'\n",
      " 'ITEM_INVOICE_CARGO_VALUE_FOREIGN_CURRENCY' 'FOREIGN_CURRENCY'\n",
      " 'EXCHANGE_RATE' 'PREFERENCE_CODE' 'ITEM_NO' 'EPC_CODE' 'APC_CODE'\n",
      " 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT_DESC' 'USED_YN' 'QUANTITY'\n",
      " 'UNIT_OF_MEASURE' 'TOTAL_FREIGHT_AMOUNT' 'ITEM_G_WEIGHT' 'ITEM_N_WEIGHT'\n",
      " 'ITEM_CARGO_VALUE_STATISTICS' 'ITEM_CUSTOMS_CARGO_VALUE' 'IMPORT_DUTY'\n",
      " 'IMPORT_DUTY_EXEMPTED' 'EXPORT_DUTY' 'EXPORT_DUTY_EXEMPTED' 'EXCISE_DUTY'\n",
      " 'EXCISE_DUTY_EXEMPTED' 'VAT' 'VAT_EXEMPTED' 'FUEL_LEVY' 'ELV' 'EXP'\n",
      " 'VEH_REG' 'RAILWAY_DEV_LEVY' 'CUSTOMS_PROCESS_FEE' 'DECLARED_CUSTOMS_VAL'\n",
      " 'ASSESSED_CUSTOMS_VAL' 'ITEM_TAX_AMOUNT' 'dataid' 'src_country' 'iesign'\n",
      " 'version']\n",
      "2025-09-10 06:49:40,487 - Data Mapping - INFO - Export data mapping and cleaning takes 0.04 minutes\n",
      "2025-09-10 06:49:40,670 - Bash Mapping - INFO - target_tab shape: (128934, 47)\n",
      "2025-09-10 06:49:41,033 - Data Mapping - INFO - imp_df columns: ['REG_DATE' 'CUSTOMS_OFFICE_CODE' 'DECLARATION_NUMBER' 'ITEM_NO' 'HS_CODE'\n",
      " 'PRODUCT_DESC' 'CUSTOMS_OFFICE_NAME_3WEI' 'SHIPPER_CODE' 'SHIPPER_NAME'\n",
      " 'GUARANTEE_CODE' 'GUARANTEE_NAME_ADDRESS' 'DEPARTURE_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'DEST_COUNTRY' 'CONSIGNEE_NAME_ADDRESS'\n",
      " 'CONTRACT_HOLDER_NAME' 'PLACE_OF_RECEIPT' 'INCOTERMS'\n",
      " 'TRANS_TYPE_AT_BORDER' 'TRANS_TYPE_WITHIN_COUNTRY' 'UOM' 'QTY'\n",
      " 'N_WEIGHT_IN_KG' 'INVOICE_VALUE_USD' 'CUSTOMS_VALUE_USD'\n",
      " 'DECLARATION_TYPE_CODE' 'DIRECTION' 'PREV_CUSTOMS_MODE_CODE'\n",
      " 'CUSTOMS_MODE' 'dataid' 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:49:41,870 - Data Mapping - INFO - Export data mapping and cleaning takes 0.06 minutes\n",
      "2025-09-10 06:49:42,028 - Bash Mapping - INFO - target_tab shape: (134581, 47)\n",
      "2025-09-10 06:49:42,935 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 06:49:43,041 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/智利/export_paths/202505-EXP.csv\n",
      "2025-09-10 06:49:43,642 - Data Mapping - INFO - imp_df columns: ['DAY' 'MONTH' 'YEAR' 'CONTROL_ID' 'CUSTOMS' 'EXPORTER_ID' 'VERIFIER'\n",
      " 'EXPORTER' 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT' 'VARIETEY' 'BRAND' 'DESC'\n",
      " 'DEST_COUNTRY' 'TRANS_TYPE' 'TRANS_CORP' 'SHIP_NAME' 'TYPE_OF_LOAD'\n",
      " 'ORIGIN_PORT' 'DEST_PORT' 'G_WEIGHT' 'QUANTITY' 'UNIT_OF_QUANTITY' 'FOB'\n",
      " 'FREIGHT' 'INSURANCE' 'CIF' 'FOB_UNIT' 'TYPE_OF_PACKAGE'\n",
      " 'EXPORTER_REGION' 'PACKAGES' 'PACKAGES_DESC' 'TRANS_CORP_COUNTRY'\n",
      " 'SALE_CONDITION' 'ECONOMIC_ZONE' 'EXPORTER_ECONOMIC_KEY'\n",
      " 'TRANS_DOC_NUMBER' 'TRANS_DOC_DATE' 'VOYAGE_NUMBER' 'INCOTERMS' 'PAYMENT'\n",
      " 'dataid' 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:49:43,893 - Data Mapping - INFO - Export data mapping and cleaning takes 0.01 minutes\n",
      "2025-09-10 06:49:43,973 - Bash Mapping - INFO - target_tab shape: (91874, 47)\n",
      "2025-09-10 06:49:45,096 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:49:45,189 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202504-EXP-RAW.csv\n",
      "2025-09-10 06:49:45,265 - Data Mapping - INFO - imp_df columns: ['CUSTOMS_OFFICE_CODE' 'TANSAD_YY' 'TANSAD_DATE' 'TANSAD_SERIAL_NO'\n",
      " 'TANSAD_FULL_NO' 'REGIME_NAME' 'TANSAD_YYYYMM' 'RELEASE_YYYYMM'\n",
      " 'PROCESSING_STATUS_NAME' 'ORIGIN_COUNTRY' 'CONSIGNMENT_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'EXPORT_COUNTRY' 'DEST_COUNTRY' 'CRN'\n",
      " 'MANIFEST_SUBMISSION_DATE' 'BL_NO' 'IS_CONTAINERIZED' 'NO_OF_CONTAINERS'\n",
      " 'VESSEL_NAME' 'TRANSPORT_MEANS_ID' 'TRANSPORT_NATIONALITY_CD'\n",
      " 'ARRIVAL_DATE' 'DISCHARGE_DATE' 'ENTRY_OFFICE_CODE'\n",
      " 'SELECTIVITY_RESULT_CD' 'CLEARANCE_PLAN_CD' 'TRANSPORT_MODE_CODE'\n",
      " 'TRANSPORT_MODE_NAME' 'IMPORTER_TAX_ID' 'IMPORTER_NAME' 'EXPORTER_TAX_ID'\n",
      " 'EXPORTER_NAME' 'DECLARANT_TAX_ID' 'DECLARANT_NAME'\n",
      " 'ITEM_INVOICE_CARGO_VALUE_FOREIGN_CURRENCY' 'FOREIGN_CURRENCY'\n",
      " 'EXCHANGE_RATE' 'PREFERENCE_CODE' 'ITEM_NO' 'EPC_CODE' 'APC_CODE'\n",
      " 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT_DESC' 'USED_YN' 'QUANTITY'\n",
      " 'UNIT_OF_MEASURE' 'TOTAL_FREIGHT_AMOUNT' 'ITEM_G_WEIGHT' 'ITEM_N_WEIGHT'\n",
      " 'ITEM_CARGO_VALUE_STATISTICS' 'ITEM_CUSTOMS_CARGO_VALUE' 'IMPORT_DUTY'\n",
      " 'IMPORT_DUTY_EXEMPTED' 'EXPORT_DUTY' 'EXPORT_DUTY_EXEMPTED' 'EXCISE_DUTY'\n",
      " 'EXCISE_DUTY_EXEMPTED' 'VAT' 'VAT_EXEMPTED' 'FUEL_LEVY' 'ELV' 'EXP'\n",
      " 'VEH_REG' 'RAILWAY_DEV_LEVY' 'CUSTOMS_PROCESS_FEE' 'DECLARED_CUSTOMS_VAL'\n",
      " 'ASSESSED_CUSTOMS_VAL' 'ITEM_TAX_AMOUNT' 'dataid' 'src_country' 'iesign'\n",
      " 'version']\n",
      "2025-09-10 06:49:45,299 - Data Mapping - INFO - Export data mapping and cleaning takes 0.0 minutes\n",
      "2025-09-10 06:49:45,311 - Bash Mapping - INFO - target_tab shape: (6534, 47)\n",
      "2025-09-10 06:49:45,467 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 06:49:45,537 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202505-EXP-RAW.csv\n",
      "2025-09-10 06:49:45,596 - Data Mapping - INFO - imp_df columns: ['CUSTOMS_OFFICE_CODE' 'TANSAD_YY' 'TANSAD_DATE' 'TANSAD_SERIAL_NO'\n",
      " 'TANSAD_FULL_NO' 'REGIME_NAME' 'TANSAD_YYYYMM' 'RELEASE_YYYYMM'\n",
      " 'PROCESSING_STATUS_NAME' 'ORIGIN_COUNTRY' 'CONSIGNMENT_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'EXPORT_COUNTRY' 'DEST_COUNTRY' 'CRN'\n",
      " 'MANIFEST_SUBMISSION_DATE' 'BL_NO' 'IS_CONTAINERIZED' 'NO_OF_CONTAINERS'\n",
      " 'VESSEL_NAME' 'TRANSPORT_MEANS_ID' 'TRANSPORT_NATIONALITY_CD'\n",
      " 'ARRIVAL_DATE' 'DISCHARGE_DATE' 'ENTRY_OFFICE_CODE'\n",
      " 'SELECTIVITY_RESULT_CD' 'CLEARANCE_PLAN_CD' 'TRANSPORT_MODE_CODE'\n",
      " 'TRANSPORT_MODE_NAME' 'IMPORTER_TAX_ID' 'IMPORTER_NAME' 'EXPORTER_TAX_ID'\n",
      " 'EXPORTER_NAME' 'DECLARANT_TAX_ID' 'DECLARANT_NAME'\n",
      " 'ITEM_INVOICE_CARGO_VALUE_FOREIGN_CURRENCY' 'FOREIGN_CURRENCY'\n",
      " 'EXCHANGE_RATE' 'PREFERENCE_CODE' 'ITEM_NO' 'EPC_CODE' 'APC_CODE'\n",
      " 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT_DESC' 'USED_YN' 'QUANTITY'\n",
      " 'UNIT_OF_MEASURE' 'TOTAL_FREIGHT_AMOUNT' 'ITEM_G_WEIGHT' 'ITEM_N_WEIGHT'\n",
      " 'ITEM_CARGO_VALUE_STATISTICS' 'ITEM_CUSTOMS_CARGO_VALUE' 'IMPORT_DUTY'\n",
      " 'IMPORT_DUTY_EXEMPTED' 'EXPORT_DUTY' 'EXPORT_DUTY_EXEMPTED' 'EXCISE_DUTY'\n",
      " 'EXCISE_DUTY_EXEMPTED' 'VAT' 'VAT_EXEMPTED' 'FUEL_LEVY' 'ELV' 'EXP'\n",
      " 'VEH_REG' 'RAILWAY_DEV_LEVY' 'CUSTOMS_PROCESS_FEE' 'DECLARED_CUSTOMS_VAL'\n",
      " 'ASSESSED_CUSTOMS_VAL' 'ITEM_TAX_AMOUNT' 'dataid' 'src_country' 'iesign'\n",
      " 'version']\n",
      "2025-09-10 06:49:45,621 - Data Mapping - INFO - Export data mapping and cleaning takes 0.0 minutes\n",
      "2025-09-10 06:49:45,632 - Bash Mapping - INFO - target_tab shape: (5331, 47)\n",
      "2025-09-10 06:49:45,760 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 06:49:45,832 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:49:45,994 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 06:49:46,078 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/智利/export_paths/202504-EXP.csv\n",
      "2025-09-10 06:49:46,968 - Data Mapping - INFO - imp_df columns: ['DAY' 'MONTH' 'YEAR' 'CONTROL_ID' 'CUSTOMS' 'EXPORTER_ID' 'VERIFIER'\n",
      " 'EXPORTER' 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT' 'VARIETEY' 'BRAND' 'DESC'\n",
      " 'DEST_COUNTRY' 'TRANS_TYPE' 'TRANS_CORP' 'SHIP_NAME' 'TYPE_OF_LOAD'\n",
      " 'ORIGIN_PORT' 'DEST_PORT' 'G_WEIGHT' 'QUANTITY' 'UNIT_OF_QUANTITY' 'FOB'\n",
      " 'FREIGHT' 'INSURANCE' 'CIF' 'FOB_UNIT' 'TYPE_OF_PACKAGE'\n",
      " 'EXPORTER_REGION' 'PACKAGES' 'PACKAGES_DESC' 'TRANS_CORP_COUNTRY'\n",
      " 'SALE_CONDITION' 'ECONOMIC_ZONE' 'EXPORTER_ECONOMIC_KEY'\n",
      " 'TRANS_DOC_NUMBER' 'TRANS_DOC_DATE' 'VOYAGE_NUMBER' 'INCOTERMS' 'PAYMENT'\n",
      " 'dataid' 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:49:47,322 - Data Mapping - INFO - Export data mapping and cleaning takes 0.02 minutes\n",
      "2025-09-10 06:49:47,412 - Bash Mapping - INFO - target_tab shape: (112769, 47)\n",
      "2025-09-10 06:49:47,851 - Data Mapping - INFO - imp_df columns: ['CUSTOMS_OFFICE_CODE' 'TANSAD_YY' 'TANSAD_DATE' 'TANSAD_SERIAL_NO'\n",
      " 'TANSAD_FULL_NO' 'REGIME_NAME' 'TANSAD_YYYYMM' 'RELEASE_YYYYMM'\n",
      " 'PROCESSING_STATUS_NAME' 'ORIGIN_COUNTRY' 'CONSIGNMENT_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'EXPORT_COUNTRY' 'DEST_COUNTRY' 'CRN'\n",
      " 'MANIFEST_SUBMISSION_DATE' 'BL_NO' 'IS_CONTAINERIZED' 'NO_OF_CONTAINERS'\n",
      " 'VESSEL_NAME' 'TRANSPORT_MEANS_ID' 'TRANSPORT_NATIONALITY_CD'\n",
      " 'ARRIVAL_DATE' 'DISCHARGE_DATE' 'ENTRY_OFFICE_CODE'\n",
      " 'SELECTIVITY_RESULT_CD' 'CLEARANCE_PLAN_CD' 'TRANSPORT_MODE_CODE'\n",
      " 'TRANSPORT_MODE_NAME' 'IMPORTER_TAX_ID' 'IMPORTER_NAME' 'EXPORTER_TAX_ID'\n",
      " 'EXPORTER_NAME' 'DECLARANT_TAX_ID' 'DECLARANT_NAME'\n",
      " 'ITEM_INVOICE_CARGO_VALUE_FOREIGN_CURRENCY' 'FOREIGN_CURRENCY'\n",
      " 'EXCHANGE_RATE' 'PREFERENCE_CODE' 'ITEM_NO' 'EPC_CODE' 'APC_CODE'\n",
      " 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT_DESC' 'USED_YN' 'QUANTITY'\n",
      " 'UNIT_OF_MEASURE' 'TOTAL_FREIGHT_AMOUNT' 'ITEM_G_WEIGHT' 'ITEM_N_WEIGHT'\n",
      " 'ITEM_CARGO_VALUE_STATISTICS' 'ITEM_CUSTOMS_CARGO_VALUE' 'IMPORT_DUTY'\n",
      " 'IMPORT_DUTY_EXEMPTED' 'EXPORT_DUTY' 'EXPORT_DUTY_EXEMPTED' 'EXCISE_DUTY'\n",
      " 'EXCISE_DUTY_EXEMPTED' 'VAT' 'VAT_EXEMPTED' 'FUEL_LEVY' 'ELV' 'EXP'\n",
      " 'VEH_REG' 'RAILWAY_DEV_LEVY' 'CUSTOMS_PROCESS_FEE' 'DECLARED_CUSTOMS_VAL'\n",
      " 'ASSESSED_CUSTOMS_VAL' 'ITEM_TAX_AMOUNT' 'dataid' 'src_country' 'iesign'\n",
      " 'version']\n",
      "2025-09-10 06:49:48,204 - Data Mapping - INFO - Export data mapping and cleaning takes 0.04 minutes\n",
      "2025-09-10 06:49:48,323 - Bash Mapping - INFO - target_tab shape: (138793, 47)\n",
      "2025-09-10 06:49:50,301 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 06:49:50,388 - Bash Mapping - INFO - all data takes 0.2 minutes\n",
      "2025-09-10 06:49:50,395 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:49:51,594 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:49:51,680 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202506-EXP-RAW.csv\n",
      "2025-09-10 06:49:51,718 - Data Mapping - INFO - imp_df columns: ['CUSTOMS_OFFICE_CODE' 'TANSAD_YY' 'TANSAD_DATE' 'TANSAD_SERIAL_NO'\n",
      " 'TANSAD_FULL_NO' 'REGIME_NAME' 'TANSAD_YYYYMM' 'RELEASE_YYYYMM'\n",
      " 'PROCESSING_STATUS_NAME' 'ORIGIN_COUNTRY' 'CONSIGNMENT_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'EXPORT_COUNTRY' 'DEST_COUNTRY' 'CRN'\n",
      " 'MANIFEST_SUBMISSION_DATE' 'BL_NO' 'IS_CONTAINERIZED' 'NO_OF_CONTAINERS'\n",
      " 'VESSEL_NAME' 'TRANSPORT_MEANS_ID' 'TRANSPORT_NATIONALITY_CD'\n",
      " 'ARRIVAL_DATE' 'DISCHARGE_DATE' 'ENTRY_OFFICE_CODE'\n",
      " 'SELECTIVITY_RESULT_CD' 'CLEARANCE_PLAN_CD' 'TRANSPORT_MODE_CODE'\n",
      " 'TRANSPORT_MODE_NAME' 'IMPORTER_TAX_ID' 'IMPORTER_NAME' 'EXPORTER_TAX_ID'\n",
      " 'EXPORTER_NAME' 'DECLARANT_TAX_ID' 'DECLARANT_NAME'\n",
      " 'ITEM_INVOICE_CARGO_VALUE_FOREIGN_CURRENCY' 'FOREIGN_CURRENCY'\n",
      " 'EXCHANGE_RATE' 'PREFERENCE_CODE' 'ITEM_NO' 'EPC_CODE' 'APC_CODE'\n",
      " 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT_DESC' 'USED_YN' 'QUANTITY'\n",
      " 'UNIT_OF_MEASURE' 'TOTAL_FREIGHT_AMOUNT' 'ITEM_G_WEIGHT' 'ITEM_N_WEIGHT'\n",
      " 'ITEM_CARGO_VALUE_STATISTICS' 'ITEM_CUSTOMS_CARGO_VALUE' 'IMPORT_DUTY'\n",
      " 'IMPORT_DUTY_EXEMPTED' 'EXPORT_DUTY' 'EXPORT_DUTY_EXEMPTED' 'EXCISE_DUTY'\n",
      " 'EXCISE_DUTY_EXEMPTED' 'VAT' 'VAT_EXEMPTED' 'FUEL_LEVY' 'ELV' 'EXP'\n",
      " 'VEH_REG' 'RAILWAY_DEV_LEVY' 'CUSTOMS_PROCESS_FEE' 'DECLARED_CUSTOMS_VAL'\n",
      " 'ASSESSED_CUSTOMS_VAL' 'ITEM_TAX_AMOUNT' 'dataid' 'src_country' 'iesign'\n",
      " 'version']\n",
      "2025-09-10 06:49:51,734 - Data Mapping - INFO - Export data mapping and cleaning takes 0.0 minutes\n",
      "2025-09-10 06:49:51,738 - Bash Mapping - INFO - target_tab shape: (5108, 47)\n",
      "2025-09-10 06:49:51,759 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:49:51,853 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/乌克兰/export_paths/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:49:51,877 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 06:49:51,943 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:49:53,208 - Data Mapping - INFO - imp_df columns: ['CUSTOMS_OFFICE_CODE' 'TANSAD_YY' 'TANSAD_DATE' 'TANSAD_SERIAL_NO'\n",
      " 'TANSAD_FULL_NO' 'REGIME_NAME' 'TANSAD_YYYYMM' 'RELEASE_YYYYMM'\n",
      " 'PROCESSING_STATUS_NAME' 'ORIGIN_COUNTRY' 'CONSIGNMENT_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'EXPORT_COUNTRY' 'DEST_COUNTRY' 'CRN'\n",
      " 'MANIFEST_SUBMISSION_DATE' 'BL_NO' 'IS_CONTAINERIZED' 'NO_OF_CONTAINERS'\n",
      " 'VESSEL_NAME' 'TRANSPORT_MEANS_ID' 'TRANSPORT_NATIONALITY_CD'\n",
      " 'ARRIVAL_DATE' 'DISCHARGE_DATE' 'ENTRY_OFFICE_CODE'\n",
      " 'SELECTIVITY_RESULT_CD' 'CLEARANCE_PLAN_CD' 'TRANSPORT_MODE_CODE'\n",
      " 'TRANSPORT_MODE_NAME' 'IMPORTER_TAX_ID' 'IMPORTER_NAME' 'EXPORTER_TAX_ID'\n",
      " 'EXPORTER_NAME' 'DECLARANT_TAX_ID' 'DECLARANT_NAME'\n",
      " 'ITEM_INVOICE_CARGO_VALUE_FOREIGN_CURRENCY' 'FOREIGN_CURRENCY'\n",
      " 'EXCHANGE_RATE' 'PREFERENCE_CODE' 'ITEM_NO' 'EPC_CODE' 'APC_CODE'\n",
      " 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT_DESC' 'USED_YN' 'QUANTITY'\n",
      " 'UNIT_OF_MEASURE' 'TOTAL_FREIGHT_AMOUNT' 'ITEM_G_WEIGHT' 'ITEM_N_WEIGHT'\n",
      " 'ITEM_CARGO_VALUE_STATISTICS' 'ITEM_CUSTOMS_CARGO_VALUE' 'IMPORT_DUTY'\n",
      " 'IMPORT_DUTY_EXEMPTED' 'EXPORT_DUTY' 'EXPORT_DUTY_EXEMPTED' 'EXCISE_DUTY'\n",
      " 'EXCISE_DUTY_EXEMPTED' 'VAT' 'VAT_EXEMPTED' 'FUEL_LEVY' 'ELV' 'EXP'\n",
      " 'VEH_REG' 'RAILWAY_DEV_LEVY' 'CUSTOMS_PROCESS_FEE' 'DECLARED_CUSTOMS_VAL'\n",
      " 'ASSESSED_CUSTOMS_VAL' 'ITEM_TAX_AMOUNT' 'dataid' 'src_country' 'iesign'\n",
      " 'version']\n",
      "2025-09-10 06:49:53,467 - Data Mapping - INFO - Export data mapping and cleaning takes 0.03 minutes\n",
      "2025-09-10 06:49:53,558 - Bash Mapping - INFO - target_tab shape: (125615, 47)\n",
      "2025-09-10 06:49:53,756 - Data Mapping - INFO - imp_df columns: ['REG_DATE' 'CUSTOMS_OFFICE_CODE' 'DECLARATION_NUMBER' 'ITEM_NO' 'HS_CODE'\n",
      " 'PRODUCT_DESC' 'CUSTOMS_OFFICE_NAME_3WEI' 'SHIPPER_CODE' 'SHIPPER_NAME'\n",
      " 'GUARANTEE_CODE' 'GUARANTEE_NAME_ADDRESS' 'DEPARTURE_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'DEST_COUNTRY' 'CONSIGNEE_NAME_ADDRESS'\n",
      " 'CONTRACT_HOLDER_NAME' 'PLACE_OF_RECEIPT' 'INCOTERMS'\n",
      " 'TRANS_TYPE_AT_BORDER' 'TRANS_TYPE_WITHIN_COUNTRY' 'UOM' 'QTY'\n",
      " 'N_WEIGHT_IN_KG' 'INVOICE_VALUE_USD' 'CUSTOMS_VALUE_USD'\n",
      " 'DECLARATION_TYPE_CODE' 'DIRECTION' 'PREV_CUSTOMS_MODE_CODE'\n",
      " 'CUSTOMS_MODE' 'dataid' 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:49:54,458 - Data Mapping - INFO - Export data mapping and cleaning takes 0.04 minutes\n",
      "2025-09-10 06:49:54,594 - Bash Mapping - INFO - target_tab shape: (135921, 47)\n",
      "2025-09-10 06:49:56,299 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:49:56,410 - Bash Mapping - INFO - all data takes 0.3 minutes\n",
      "2025-09-10 06:49:56,411 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:49:57,847 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:49:57,975 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/乌克兰/export_paths/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:49:58,874 - Data Mapping - INFO - imp_df columns: ['REG_DATE' 'CUSTOMS_OFFICE_CODE' 'DECLARATION_NUMBER' 'ITEM_NO' 'HS_CODE'\n",
      " 'PRODUCT_DESC' 'CUSTOMS_OFFICE_NAME_3WEI' 'SHIPPER_CODE' 'SHIPPER_NAME'\n",
      " 'GUARANTEE_CODE' 'GUARANTEE_NAME_ADDRESS' 'DEPARTURE_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'DEST_COUNTRY' 'CONSIGNEE_NAME_ADDRESS'\n",
      " 'CONTRACT_HOLDER_NAME' 'PLACE_OF_RECEIPT' 'INCOTERMS'\n",
      " 'TRANS_TYPE_AT_BORDER' 'TRANS_TYPE_WITHIN_COUNTRY' 'UOM' 'QTY'\n",
      " 'N_WEIGHT_IN_KG' 'INVOICE_VALUE_USD' 'CUSTOMS_VALUE_USD'\n",
      " 'DECLARATION_TYPE_CODE' 'DIRECTION' 'PREV_CUSTOMS_MODE_CODE'\n",
      " 'CUSTOMS_MODE' 'dataid' 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:49:59,516 - Data Mapping - INFO - Export data mapping and cleaning takes 0.03 minutes\n",
      "2025-09-10 06:49:59,653 - Bash Mapping - INFO - target_tab shape: (147189, 47)\n",
      "2025-09-10 06:50:02,043 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:50:02,189 - Bash Mapping - INFO - all data takes 0.4 minutes\n",
      "2025-09-10 06:50:02,190 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:50:02,191 - Data Goverance - INFO - country_normal_mapping takes 0.3994826912879944 minutes\n"
     ]
    }
   ],
   "source": [
    "# 出口跑批\n",
    "start = time.time()\n",
    "\n",
    "params_list = [([ctry], root_dir, csv_files_path, all_country_cols_dict) for ctry in country_list]\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: bash_mapping_export(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        logger.info(future.result())\n",
    "        # pass\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'country_normal_mapping takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c960344b-a014-448c-9d24-221c07d668eb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93098180-1531-43b7-87b9-3ce668e685ab",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "57029054-4736-4783-a7d6-3418e0346157",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "### 5.2.2 进口跑批"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "48dec95f-ee93-4b11-937c-5d7ec4fb16b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取文件目录，区分出口、进口数据\n",
    "root_dir = Path('dataset/data_20250724/20254t6_src_version')\n",
    "\n",
    "# csv结果文件保存路径\n",
    "# csv_files_path = Path('dataset/data_20250724/20251t3_csv_save')\n",
    "csv_files_path = Path('dataset/data_20250724/20254t6_csv_save')\n",
    "\n",
    "country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# country_list = ['智利']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e2fe8ada-a656-4396-8ffd-c6eafb1050e4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:50:02,210 - Bash Mapping - INFO - 智利\n",
      "2025-09-10 06:50:02,211 - Bash Mapping - INFO - 坦桑尼亚\n",
      "2025-09-10 06:50:02,211 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/智利/import_paths/202504-IMP.csv\n",
      "2025-09-10 06:50:02,211 - Bash Mapping - INFO - 乌克兰\n",
      "2025-09-10 06:50:02,212 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/坦桑尼亚/import_paths/202504-IMP-RAW.csv\n",
      "2025-09-10 06:50:02,212 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/乌克兰/import_paths/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:50:06,356 - Data Mapping - INFO - imp_df columns: ['DAY' 'MONTH' 'YEAR' 'CUSTOMS' 'CONTROL_ID' 'IMPORTER_ID' 'VERIFIER'\n",
      " 'IMPORTER' 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT' 'BRAND' 'VARIETEY' 'DESC'\n",
      " 'ORIGIN_COUNTRY' 'SALES_COUNTRY' 'TRANS_TYPE' 'PAYMENT' 'ORIGIN_PORT'\n",
      " 'DEST_PORT' 'TRANS_CORP' 'TYPE_OF_LOAD' 'TYPE_OF_PACKAGE' 'G_WEIGHT'\n",
      " 'INCOTERMS' 'TAX' 'QUANTITY' 'UNIT_OF_QUANTITY' 'FOB' 'FREIGHT'\n",
      " 'INSURANCE' 'CIF' 'CIF_UNIT' 'FOB_UNIT' 'TRANS_CORP_COUNTRY' 'US_TAX'\n",
      " 'PACKAGES' 'ECONOMIC_ZONE' 'IMPORTER_ECONOMIC_KEY' 'MANIFEST_NUMBER'\n",
      " 'MANIFEST_DATE' 'TRANS_DOC_NUMBER' 'TRANS_DOC_DATE' 'dataid'\n",
      " 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:50:07,463 - Data Mapping - INFO - Import data mapping and cleaning takes 0.09 minutes\n",
      "2025-09-10 06:50:07,931 - Bash Mapping - INFO - target_tab shape: (390891, 47)\n",
      "2025-09-10 06:50:09,024 - Data Mapping - INFO - imp_df columns: ['CUSTOMS_OFFICE_CODE' 'TANSAD_YY' 'TANSAD_DATE' 'TANSAD_SERIAL_NO'\n",
      " 'TANSAD_FULL_NO' 'REGIME_NAME' 'TANSAD_YYYYMM' 'RELEASE_YYYYMM'\n",
      " 'PROCESSING_STATUS_NAME' 'ORIGIN_COUNTRY' 'CONSIGNMENT_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'EXPORT_COUNTRY' 'DEST_COUNTRY' 'CRN'\n",
      " 'MANIFEST_SUBMISSION_DATE' 'BL_NO' 'IS_CONTAINERIZED' 'NO_OF_CONTAINERS'\n",
      " 'VESSEL_NAME' 'TRANSPORT_MEANS_ID' 'TRANSPORT_NATIONALITY_CD'\n",
      " 'ARRIVAL_DATE' 'DISCHARGE_DATE' 'ENTRY_OFFICE_CODE'\n",
      " 'SELECTIVITY_RESULT_CD' 'CLEARANCE_PLAN_CD' 'TRANSPORT_MODE_CODE'\n",
      " 'TRANSPORT_MODE_NAME' 'IMPORTER_TAX_ID' 'IMPORTER_NAME' 'EXPORTER_TAX_ID'\n",
      " 'EXPORTER_NAME' 'DECLARANT_TAX_ID' 'DECLARANT_NAME'\n",
      " 'ITEM_INVOICE_CARGO_VALUE_FOREIGN_CURRENCY' 'FOREIGN_CURRENCY'\n",
      " 'EXCHANGE_RATE' 'PREFERENCE_CODE' 'ITEM_NO' 'EPC_CODE' 'APC_CODE'\n",
      " 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT_DESC' 'USED_YN' 'QUANTITY'\n",
      " 'UNIT_OF_MEASURE' 'TOTAL_FREIGHT_AMOUNT' 'ITEM_G_WEIGHT' 'ITEM_N_WEIGHT'\n",
      " 'ITEM_CARGO_VALUE_STATISTICS' 'ITEM_CUSTOMS_CARGO_VALUE' 'IMPORT_DUTY'\n",
      " 'IMPORT_DUTY_EXEMPTED' 'EXPORT_DUTY' 'EXPORT_DUTY_EXEMPTED' 'EXCISE_DUTY'\n",
      " 'EXCISE_DUTY_EXEMPTED' 'VAT' 'VAT_EXEMPTED' 'FUEL_LEVY' 'ELV' 'EXP'\n",
      " 'VEH_REG' 'RAILWAY_DEV_LEVY' 'CUSTOMS_PROCESS_FEE' 'DECLARED_CUSTOMS_VAL'\n",
      " 'ASSESSED_CUSTOMS_VAL' 'ITEM_TAX_AMOUNT' 'dataid' 'src_country' 'iesign'\n",
      " 'version']\n",
      "2025-09-10 06:50:09,947 - Data Mapping - INFO - Import data mapping and cleaning takes 0.13 minutes\n",
      "2025-09-10 06:50:10,339 - Bash Mapping - INFO - target_tab shape: (387273, 47)\n",
      "2025-09-10 06:50:14,171 - Data Mapping - INFO - imp_df columns: ['REG_DATE' 'CUSTOMS_OFFICE_CODE' 'DECLARATION_NUMBER' 'ITEM_NO' 'HS_CODE'\n",
      " 'PRODUCT_DESC' 'CUSTOMS_OFFICE_NAME_3WEI' 'CONSIGNEE_CODE'\n",
      " 'CONSIGNEE_NAME_ADDRESS' 'GUARANTEE_CODE' 'GUARANTEE_NAME_ADDRESS'\n",
      " 'DEPARTURE_COUNTRY' 'TRADING_COUNTRY' 'ORIGIN_COUNTRY' 'SHIPPER_NAME'\n",
      " 'CONTRACT_HOLDER_NAME' 'PLACE_OF_DELIVERY' 'INCOTERMS'\n",
      " 'TRANS_TYPE_AT_BORDER' 'TRANS_TYPE_WITHIN_COUNTRY' 'UOM' 'QTY'\n",
      " 'N_WEIGHT_IN_KG' 'INVOICE_VALUE_USD' 'CUSTOMS_VALUE_USD'\n",
      " 'DECLARATION_TYPE_CODE' 'DIRECTION' 'PREV_CUSTOMS_MODE_CODE'\n",
      " 'CUSTOMS_MODE' 'dataid' 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:50:16,982 - Data Mapping - INFO - Import data mapping and cleaning takes 0.25 minutes\n",
      "2025-09-10 06:50:17,835 - Bash Mapping - INFO - target_tab shape: (739143, 47)\n",
      "2025-09-10 06:50:22,739 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 06:50:22,955 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/坦桑尼亚/import_paths/202506-IMP-RAW.csv\n",
      "2025-09-10 06:50:23,146 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 06:50:23,254 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/智利/import_paths/202505-IMP.csv\n",
      "2025-09-10 06:50:26,726 - Data Mapping - INFO - imp_df columns: ['DAY' 'MONTH' 'YEAR' 'CUSTOMS' 'CONTROL_ID' 'IMPORTER_ID' 'VERIFIER'\n",
      " 'IMPORTER' 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT' 'BRAND' 'VARIETEY' 'DESC'\n",
      " 'ORIGIN_COUNTRY' 'SALES_COUNTRY' 'TRANS_TYPE' 'PAYMENT' 'ORIGIN_PORT'\n",
      " 'DEST_PORT' 'TRANS_CORP' 'TYPE_OF_LOAD' 'TYPE_OF_PACKAGE' 'G_WEIGHT'\n",
      " 'INCOTERMS' 'TAX' 'QUANTITY' 'UNIT_OF_QUANTITY' 'FOB' 'FREIGHT'\n",
      " 'INSURANCE' 'CIF' 'CIF_UNIT' 'FOB_UNIT' 'TRANS_CORP_COUNTRY' 'US_TAX'\n",
      " 'PACKAGES' 'ECONOMIC_ZONE' 'IMPORTER_ECONOMIC_KEY' 'MANIFEST_NUMBER'\n",
      " 'MANIFEST_DATE' 'TRANS_DOC_NUMBER' 'TRANS_DOC_DATE' 'dataid'\n",
      " 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:50:27,842 - Data Mapping - INFO - Import data mapping and cleaning takes 0.08 minutes\n",
      "2025-09-10 06:50:28,251 - Bash Mapping - INFO - target_tab shape: (404234, 47)\n",
      "2025-09-10 06:50:29,140 - Data Mapping - INFO - imp_df columns: ['CUSTOMS_OFFICE_CODE' 'TANSAD_YY' 'TANSAD_DATE' 'TANSAD_SERIAL_NO'\n",
      " 'TANSAD_FULL_NO' 'REGIME_NAME' 'TANSAD_YYYYMM' 'RELEASE_YYYYMM'\n",
      " 'PROCESSING_STATUS_NAME' 'ORIGIN_COUNTRY' 'CONSIGNMENT_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'EXPORT_COUNTRY' 'DEST_COUNTRY' 'CRN'\n",
      " 'MANIFEST_SUBMISSION_DATE' 'BL_NO' 'IS_CONTAINERIZED' 'NO_OF_CONTAINERS'\n",
      " 'VESSEL_NAME' 'TRANSPORT_MEANS_ID' 'TRANSPORT_NATIONALITY_CD'\n",
      " 'ARRIVAL_DATE' 'DISCHARGE_DATE' 'ENTRY_OFFICE_CODE'\n",
      " 'SELECTIVITY_RESULT_CD' 'CLEARANCE_PLAN_CD' 'TRANSPORT_MODE_CODE'\n",
      " 'TRANSPORT_MODE_NAME' 'IMPORTER_TAX_ID' 'IMPORTER_NAME' 'EXPORTER_TAX_ID'\n",
      " 'EXPORTER_NAME' 'DECLARANT_TAX_ID' 'DECLARANT_NAME'\n",
      " 'ITEM_INVOICE_CARGO_VALUE_FOREIGN_CURRENCY' 'FOREIGN_CURRENCY'\n",
      " 'EXCHANGE_RATE' 'PREFERENCE_CODE' 'ITEM_NO' 'EPC_CODE' 'APC_CODE'\n",
      " 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT_DESC' 'USED_YN' 'QUANTITY'\n",
      " 'UNIT_OF_MEASURE' 'TOTAL_FREIGHT_AMOUNT' 'ITEM_G_WEIGHT' 'ITEM_N_WEIGHT'\n",
      " 'ITEM_CARGO_VALUE_STATISTICS' 'ITEM_CUSTOMS_CARGO_VALUE' 'IMPORT_DUTY'\n",
      " 'IMPORT_DUTY_EXEMPTED' 'EXPORT_DUTY' 'EXPORT_DUTY_EXEMPTED' 'EXCISE_DUTY'\n",
      " 'EXCISE_DUTY_EXEMPTED' 'VAT' 'VAT_EXEMPTED' 'FUEL_LEVY' 'ELV' 'EXP'\n",
      " 'VEH_REG' 'RAILWAY_DEV_LEVY' 'CUSTOMS_PROCESS_FEE' 'DECLARED_CUSTOMS_VAL'\n",
      " 'ASSESSED_CUSTOMS_VAL' 'ITEM_TAX_AMOUNT' 'dataid' 'src_country' 'iesign'\n",
      " 'version']\n",
      "2025-09-10 06:50:29,948 - Data Mapping - INFO - Import data mapping and cleaning takes 0.12 minutes\n",
      "2025-09-10 06:50:30,312 - Bash Mapping - INFO - target_tab shape: (344016, 47)\n",
      "2025-09-10 06:50:37,234 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 06:50:37,348 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/坦桑尼亚/import_paths/202505-IMP-RAW.csv\n",
      "2025-09-10 06:50:38,735 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 06:50:38,869 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/智利/import_paths/202506-IMP.csv\n",
      "2025-09-10 06:50:41,086 - Data Mapping - INFO - imp_df columns: ['CUSTOMS_OFFICE_CODE' 'TANSAD_YY' 'TANSAD_DATE' 'TANSAD_SERIAL_NO'\n",
      " 'TANSAD_FULL_NO' 'REGIME_NAME' 'TANSAD_YYYYMM' 'RELEASE_YYYYMM'\n",
      " 'PROCESSING_STATUS_NAME' 'ORIGIN_COUNTRY' 'CONSIGNMENT_COUNTRY'\n",
      " 'TRADING_COUNTRY' 'EXPORT_COUNTRY' 'DEST_COUNTRY' 'CRN'\n",
      " 'MANIFEST_SUBMISSION_DATE' 'BL_NO' 'IS_CONTAINERIZED' 'NO_OF_CONTAINERS'\n",
      " 'VESSEL_NAME' 'TRANSPORT_MEANS_ID' 'TRANSPORT_NATIONALITY_CD'\n",
      " 'ARRIVAL_DATE' 'DISCHARGE_DATE' 'ENTRY_OFFICE_CODE'\n",
      " 'SELECTIVITY_RESULT_CD' 'CLEARANCE_PLAN_CD' 'TRANSPORT_MODE_CODE'\n",
      " 'TRANSPORT_MODE_NAME' 'IMPORTER_TAX_ID' 'IMPORTER_NAME' 'EXPORTER_TAX_ID'\n",
      " 'EXPORTER_NAME' 'DECLARANT_TAX_ID' 'DECLARANT_NAME'\n",
      " 'ITEM_INVOICE_CARGO_VALUE_FOREIGN_CURRENCY' 'FOREIGN_CURRENCY'\n",
      " 'EXCHANGE_RATE' 'PREFERENCE_CODE' 'ITEM_NO' 'EPC_CODE' 'APC_CODE'\n",
      " 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT_DESC' 'USED_YN' 'QUANTITY'\n",
      " 'UNIT_OF_MEASURE' 'TOTAL_FREIGHT_AMOUNT' 'ITEM_G_WEIGHT' 'ITEM_N_WEIGHT'\n",
      " 'ITEM_CARGO_VALUE_STATISTICS' 'ITEM_CUSTOMS_CARGO_VALUE' 'IMPORT_DUTY'\n",
      " 'IMPORT_DUTY_EXEMPTED' 'EXPORT_DUTY' 'EXPORT_DUTY_EXEMPTED' 'EXCISE_DUTY'\n",
      " 'EXCISE_DUTY_EXEMPTED' 'VAT' 'VAT_EXEMPTED' 'FUEL_LEVY' 'ELV' 'EXP'\n",
      " 'VEH_REG' 'RAILWAY_DEV_LEVY' 'CUSTOMS_PROCESS_FEE' 'DECLARED_CUSTOMS_VAL'\n",
      " 'ASSESSED_CUSTOMS_VAL' 'ITEM_TAX_AMOUNT' 'dataid' 'src_country' 'iesign'\n",
      " 'version']\n",
      "2025-09-10 06:50:41,808 - Data Mapping - INFO - Import data mapping and cleaning takes 0.07 minutes\n",
      "2025-09-10 06:50:42,186 - Bash Mapping - INFO - target_tab shape: (310571, 47)\n",
      "2025-09-10 06:50:43,398 - Data Mapping - INFO - imp_df columns: ['DAY' 'MONTH' 'YEAR' 'CUSTOMS' 'CONTROL_ID' 'IMPORTER_ID' 'VERIFIER'\n",
      " 'IMPORTER' 'HS_CODE' 'HS_CODE_DESC' 'PRODUCT' 'BRAND' 'VARIETEY' 'DESC'\n",
      " 'ORIGIN_COUNTRY' 'SALES_COUNTRY' 'TRANS_TYPE' 'PAYMENT' 'ORIGIN_PORT'\n",
      " 'DEST_PORT' 'TRANS_CORP' 'TYPE_OF_LOAD' 'TYPE_OF_PACKAGE' 'G_WEIGHT'\n",
      " 'INCOTERMS' 'TAX' 'QUANTITY' 'UNIT_OF_QUANTITY' 'FOB' 'FREIGHT'\n",
      " 'INSURANCE' 'CIF' 'CIF_UNIT' 'FOB_UNIT' 'TRANS_CORP_COUNTRY' 'US_TAX'\n",
      " 'PACKAGES' 'ECONOMIC_ZONE' 'IMPORTER_ECONOMIC_KEY' 'MANIFEST_NUMBER'\n",
      " 'MANIFEST_DATE' 'TRANS_DOC_NUMBER' 'TRANS_DOC_DATE' 'dataid'\n",
      " 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:50:44,502 - Data Mapping - INFO - Import data mapping and cleaning takes 0.09 minutes\n",
      "2025-09-10 06:50:44,894 - Bash Mapping - INFO - target_tab shape: (400856, 47)\n",
      "2025-09-10 06:50:50,016 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 06:50:50,129 - Bash Mapping - INFO - all data takes 0.8 minutes\n",
      "2025-09-10 06:50:50,131 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:50:53,572 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 06:50:53,686 - Bash Mapping - INFO - all data takes 0.86 minutes\n",
      "2025-09-10 06:50:53,686 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:50:56,056 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:50:56,220 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/乌克兰/import_paths/UA_IMPORT_202506.csv\n",
      "2025-09-10 06:50:59,851 - Data Mapping - INFO - imp_df columns: ['REG_DATE' 'CUSTOMS_OFFICE_CODE' 'DECLARATION_NUMBER' 'ITEM_NO' 'HS_CODE'\n",
      " 'PRODUCT_DESC' 'CUSTOMS_OFFICE_NAME_3WEI' 'CONSIGNEE_CODE'\n",
      " 'CONSIGNEE_NAME_ADDRESS' 'GUARANTEE_CODE' 'GUARANTEE_NAME_ADDRESS'\n",
      " 'DEPARTURE_COUNTRY' 'TRADING_COUNTRY' 'ORIGIN_COUNTRY' 'SHIPPER_NAME'\n",
      " 'CONTRACT_HOLDER_NAME' 'PLACE_OF_DELIVERY' 'INCOTERMS'\n",
      " 'TRANS_TYPE_AT_BORDER' 'TRANS_TYPE_WITHIN_COUNTRY' 'UOM' 'QTY'\n",
      " 'N_WEIGHT_IN_KG' 'INVOICE_VALUE_USD' 'CUSTOMS_VALUE_USD'\n",
      " 'DECLARATION_TYPE_CODE' 'DIRECTION' 'PREV_CUSTOMS_MODE_CODE'\n",
      " 'CUSTOMS_MODE' 'dataid' 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:51:02,425 - Data Mapping - INFO - Import data mapping and cleaning takes 0.1 minutes\n",
      "2025-09-10 06:51:03,225 - Bash Mapping - INFO - target_tab shape: (744475, 47)\n",
      "2025-09-10 06:51:14,227 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 06:51:14,521 - Bash Mapping - INFO - dataset/data_20250724/20254t6_src_version/乌克兰/import_paths/UA_IMPORT_202505.csv\n",
      "2025-09-10 06:51:18,510 - Data Mapping - INFO - imp_df columns: ['REG_DATE' 'CUSTOMS_OFFICE_CODE' 'DECLARATION_NUMBER' 'ITEM_NO' 'HS_CODE'\n",
      " 'PRODUCT_DESC' 'CUSTOMS_OFFICE_NAME_3WEI' 'CONSIGNEE_CODE'\n",
      " 'CONSIGNEE_NAME_ADDRESS' 'GUARANTEE_CODE' 'GUARANTEE_NAME_ADDRESS'\n",
      " 'DEPARTURE_COUNTRY' 'TRADING_COUNTRY' 'ORIGIN_COUNTRY' 'SHIPPER_NAME'\n",
      " 'CONTRACT_HOLDER_NAME' 'PLACE_OF_DELIVERY' 'INCOTERMS'\n",
      " 'TRANS_TYPE_AT_BORDER' 'TRANS_TYPE_WITHIN_COUNTRY' 'UOM' 'QTY'\n",
      " 'N_WEIGHT_IN_KG' 'INVOICE_VALUE_USD' 'CUSTOMS_VALUE_USD'\n",
      " 'DECLARATION_TYPE_CODE' 'DIRECTION' 'PREV_CUSTOMS_MODE_CODE'\n",
      " 'CUSTOMS_MODE' 'dataid' 'src_country' 'iesign' 'version']\n",
      "2025-09-10 06:51:20,989 - Data Mapping - INFO - Import data mapping and cleaning takes 0.11 minutes\n",
      "2025-09-10 06:51:21,792 - Bash Mapping - INFO - target_tab shape: (755738, 47)\n",
      "2025-09-10 06:51:32,687 - File Process - INFO - CSV data saved to dataset/data_20250724/20254t6_csv_save/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 06:51:32,964 - Bash Mapping - INFO - all data takes 1.51 minutes\n",
      "2025-09-10 06:51:32,965 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:51:32,978 - Data Goverance - INFO - country_normal_mapping takes 1.512840501467387 minutes\n"
     ]
    }
   ],
   "source": [
    "# 进口跑批\n",
    "start = time.time()\n",
    "\n",
    "params_list = [([ctry], root_dir, csv_files_path, all_country_cols_dict) for ctry in country_list]\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: bash_mapping_import(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        logger.info(future.result())\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'country_normal_mapping takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2f27e0d-c342-438c-b148-ea7e48fd5f08",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76fcd770-9218-41f2-82db-6da08dc2a376",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "965e6f87-8153-468f-a4d2-f39ea7c658dc",
   "metadata": {},
   "source": [
    "# 6 国家标准化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "05543fd3-28a2-42d9-8d3b-dd29b4ec222e",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# country_list = ['智利']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c7f98279-5124-45b0-82af-9a0f4d94d23c",
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_files_path = Path('dataset/data_20250724/20254t6_csv_save')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "dd2117d8-fe8e-484d-85a2-b00024a24b97",
   "metadata": {},
   "outputs": [],
   "source": [
    "pth_change_tag='20254t6_csv_correct'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "23e53224-2125-43a0-b0bb-2c4c2d5f2626",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:51:32,998 - Data Goverance - INFO - country recore count is 1886\n",
      "2025-09-10 06:51:32,998 - Data Goverance - INFO - country_mapping count is 1906\n"
     ]
    }
   ],
   "source": [
    "logger.info(f'country recore count is {len(all_country_list)}')\n",
    "logger.info(f'country_mapping count is {len(country_mapping)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "595b49e9-0c7d-4ed1-b816-752bdbc5f8ec",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:51:33,003 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 06:51:33,004 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:51:33,004 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:51:37,093 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 06:51:37,098 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 06:51:38,941 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:51:38,944 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 06:51:39,180 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 06:51:39,184 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 06:51:39,361 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 06:51:39,364 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:51:39,800 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 06:51:39,801 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 06:51:43,887 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 06:51:43,891 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 06:51:44,482 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:51:44,490 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 06:51:44,664 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 06:51:44,665 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:51:45,179 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:51:45,184 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:51:51,311 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:51:51,318 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 06:51:59,690 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:51:59,692 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:52:00,290 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 06:52:00,297 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 06:52:00,612 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 06:52:00,622 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 06:52:13,196 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 06:52:13,199 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 06:52:13,861 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:52:13,862 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:52:14,669 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 06:52:14,678 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 06:52:27,887 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 06:52:27,892 - File Process - INFO - Correct country takes 55 seconds\n",
      "2025-09-10 06:52:27,939 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:52:31,401 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 06:52:31,401 - File Process - INFO - Correct country takes 58 seconds\n",
      "2025-09-10 06:52:31,457 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:52:40,231 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:52:40,232 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 06:52:57,500 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 06:52:57,503 - File Process - INFO - dataset/data_20250724/20254t6_csv_save/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 06:53:15,558 - File Process - INFO - correct country saved to dataset/data_20250724/20254t6_csv_correct/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 06:53:15,560 - File Process - INFO - Correct country takes 103 seconds\n",
      "2025-09-10 06:53:15,675 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:53:15,679 - Data Goverance - INFO - country_normal_mapping takes 1.7112435698509216 minutes\n"
     ]
    }
   ],
   "source": [
    "# 更新国家 - 多线程\n",
    "start = time.time()\n",
    "\n",
    "params_list = [(country_mapping, [ctry], csv_files_path, pth_change_tag) for ctry in country_list]\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: country_normal_mapping(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        logger.info(future.result())\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'country_normal_mapping takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "cf8c4ff1-a093-4a03-861d-aec9be1f4df0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 更新国家 - 单线程\n",
    "# country_normal_mapping(country_mapping, country_list, csv_files_path, pth_change_tag='20251t3_csv_correct')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7260d38-389a-4cee-9869-84eb787be0a2",
   "metadata": {},
   "source": [
    "# 7 货币单位校正"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ad9a8c9-7fa3-4e36-a246-251a9963494d",
   "metadata": {},
   "source": [
    "## 7.3 替换货币"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "b5e6e228-1659-402b-a412-51c9d58bd4fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "currency_on_usd_dict=currency_on_usd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5f2443fd-3149-403c-8cc5-c5b1c6d6806f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from_dir=Path('dataset/data_20250724/20254t6_csv_correct')\n",
    "to_dir=Path('dataset/data_20250724/20254t6_csv_correct1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "4256e304-0aad-434e-b472-6920160036eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# country_list = ['智利']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "545b8a69-505f-41b5-aa95-9539bdf1b3ac",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# replace_currency_to_usd(country_list, currency_on_usd_dict, from_dir, to_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "a33676ca-f163-448f-9cf7-4ccb502d1f9b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:53:15,706 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 06:53:15,707 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:53:15,707 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:53:19,293 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 06:53:19,295 - File Process - INFO - currency transform done, the data shape is (87003, 47)\n",
      "2025-09-10 06:53:19,299 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 06:53:20,976 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:53:20,982 - File Process - INFO - currency transform done, the data shape is (128934, 47)\n",
      "2025-09-10 06:53:20,984 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 06:53:21,213 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 06:53:21,217 - File Process - INFO - currency transform done, the data shape is (6534, 47)\n",
      "2025-09-10 06:53:21,220 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 06:53:21,382 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 06:53:21,383 - File Process - INFO - currency transform done, the data shape is (5331, 47)\n",
      "2025-09-10 06:53:21,384 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:53:21,940 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 06:53:21,941 - File Process - INFO - currency transform done, the data shape is (91874, 47)\n",
      "2025-09-10 06:53:21,957 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 06:53:25,597 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 06:53:25,597 - File Process - INFO - currency transform done, the data shape is (112769, 47)\n",
      "2025-09-10 06:53:25,598 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 06:53:26,129 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:53:26,131 - File Process - INFO - currency transform done, the data shape is (138793, 47)\n",
      "2025-09-10 06:53:26,132 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 06:53:26,367 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 06:53:26,368 - File Process - INFO - currency transform done, the data shape is (5108, 47)\n",
      "2025-09-10 06:53:26,368 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:53:26,567 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:53:26,591 - File Process - INFO - currency transform done, the data shape is (134581, 47)\n",
      "2025-09-10 06:53:26,591 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:53:32,116 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:53:32,122 - File Process - INFO - currency transform done, the data shape is (125615, 47)\n",
      "2025-09-10 06:53:32,123 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 06:53:38,452 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:53:38,456 - File Process - INFO - currency transform done, the data shape is (135921, 47)\n",
      "2025-09-10 06:53:38,462 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:53:39,426 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 06:53:39,435 - File Process - INFO - currency transform done, the data shape is (390891, 47)\n",
      "2025-09-10 06:53:39,450 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 06:53:44,222 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 06:53:44,223 - File Process - INFO - currency transform done, the data shape is (387273, 47)\n",
      "2025-09-10 06:53:44,225 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 06:53:50,437 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:53:50,439 - File Process - INFO - currency transform done, the data shape is (147189, 47)\n",
      "2025-09-10 06:53:50,442 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:53:53,794 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 06:53:53,811 - File Process - INFO - currency transform done, the data shape is (404234, 47)\n",
      "2025-09-10 06:53:53,812 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 06:53:54,385 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 06:53:54,399 - File Process - INFO - currency transform done, the data shape is (344016, 47)\n",
      "2025-09-10 06:53:54,399 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 06:54:06,658 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 06:54:06,660 - File Process - INFO - currency transform done, the data shape is (310571, 47)\n",
      "2025-09-10 06:54:06,660 - File Process - INFO - replace currency to USD takes 51 seconds\n",
      "2025-09-10 06:54:06,694 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:54:09,278 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 06:54:09,279 - File Process - INFO - currency transform done, the data shape is (400856, 47)\n",
      "2025-09-10 06:54:09,280 - File Process - INFO - replace currency to USD takes 54 seconds\n",
      "2025-09-10 06:54:09,324 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:54:17,854 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:54:17,855 - File Process - INFO - currency transform done, the data shape is (739143, 47)\n",
      "2025-09-10 06:54:17,855 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 06:54:34,889 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 06:54:34,889 - File Process - INFO - currency transform done, the data shape is (744475, 47)\n",
      "2025-09-10 06:54:34,889 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 06:54:53,606 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 06:54:53,607 - File Process - INFO - currency transform done, the data shape is (755738, 47)\n",
      "2025-09-10 06:54:53,607 - File Process - INFO - replace currency to USD takes 98 seconds\n",
      "2025-09-10 06:54:53,694 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:54:53,695 - Data Goverance - INFO - replace_currency_to_usd takes 1.63317365248998 minutes\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "params_list = [([ctry], currency_on_usd_dict, from_dir, to_dir) for ctry in country_list]\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: replace_currency_to_usd(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        logger.info(future.result())\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'replace_currency_to_usd takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b5ef8b8-72ad-4f92-8d1b-c9b238f7e79d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a7c43cfa-c44f-4495-9c27-7838f6ef706e",
   "metadata": {},
   "source": [
    "## 7.4 结果检查"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4fbbbddc-4a16-431f-8075-c5f3d8b201ab",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# !head -3 dataset/data_20250724/20251t3_csv_correct3/印度/EXPORT/INDIA_EXPORT_202501_2106.csv\n",
    "# !head -10 dataset/data_20250724/20251t3_csv_correct3/纳米比亚/IMPORT/202501-IMP-FULL.csv\n",
    "# !head -3 dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
    "\n",
    "# !head -3 dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "52a03cba-5b36-491c-8db8-846680f20882",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>channelno</th>\n",
       "      <th>dataid</th>\n",
       "      <th>iesign</th>\n",
       "      <th>datatype</th>\n",
       "      <th>writeoffflag</th>\n",
       "      <th>writeoffdataid</th>\n",
       "      <th>outputdate</th>\n",
       "      <th>origincountrycode</th>\n",
       "      <th>origincountry</th>\n",
       "      <th>countrycodeofdelivery</th>\n",
       "      <th>countryofdelivery</th>\n",
       "      <th>importername</th>\n",
       "      <th>importeraddress</th>\n",
       "      <th>importercontact</th>\n",
       "      <th>suppliername</th>\n",
       "      <th>supplieraddress</th>\n",
       "      <th>suppliercontact</th>\n",
       "      <th>hscode</th>\n",
       "      <th>hscodedescription</th>\n",
       "      <th>commoditydescription</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>44b5d0c1476c46be</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-08</td>\n",
       "      <td>CHN</td>\n",
       "      <td>CHINA</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>84705000</td>\n",
       "      <td>CAJAS REGISTRADORAS.</td>\n",
       "      <td>TERMINAL DE PAGOWISEASY-FN5APARATO ELECTRONICO...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>f863182cba6b4bcc</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-09</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>85176900</td>\n",
       "      <td>LOS DEMAS APARATOS PARA EMISION, TRANSMISION O...</td>\n",
       "      <td>RECEPTOR DE SENALSHUREULXD4INALAMBRICO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>9e14d8c8f06b4c86</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-10</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>71082000</td>\n",
       "      <td>ORO PARA USO MONETARIO.</td>\n",
       "      <td>MONEDA DE OROAPMEX-FICONO BEETHOVENDE 1 OZ - D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>fda26d8758334918</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-22</td>\n",
       "      <td>ESP</td>\n",
       "      <td>SPAIN</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>84715000</td>\n",
       "      <td>UNIDADES DE PROCESO DIGITALES, EXCEPTO DE LAS ...</td>\n",
       "      <td>COMPUTADORLENOVOTHINKSTATIONP8MAQUINA AUTOMATI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>cbae64933a2f4433</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-20</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>95030099</td>\n",
       "      <td>LOS DEMÃS</td>\n",
       "      <td>FIGURAS DE ACCIONBOLT ACTIONSOLDADOS INFANTERI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383007</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>e340a52adeec46f7</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-20</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>90262090</td>\n",
       "      <td>LOS DEMAS INSTRUMENTOS PARA MEDIDA O CONTROL D...</td>\n",
       "      <td>EQUIPOAR-TECH INTERNATIONAL, INC-FDE PRUEBA, M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383008</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>7329b1b93d874345</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-20</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32091019</td>\n",
       "      <td>LAS DEMAS</td>\n",
       "      <td>PINTURAJAGUAR-FANTICORROSIVOMOD. TT-P-1757YGLP...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383009</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>12d1d3b488f842ed</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-20</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>84091000</td>\n",
       "      <td>PARTES IDENTIFICABLES COMO DESTINADAS EXCLUSIV...</td>\n",
       "      <td>CIGUENALAIRPARTS COMPANY INC-FDE ACERO, MOD. 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383010</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>360d2f62531348b9</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-23</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>73262000</td>\n",
       "      <td>MANUFACTURAS DE ALAMBRE DE HIERRO O DE ACERO.</td>\n",
       "      <td>DISCOSMIDWEST AEROSPACE, LTD-FDE SUJECION, MOD...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383011</th>\n",
       "      <td>zdzj</td>\n",
       "      <td>18d98b795b844c89</td>\n",
       "      <td>I</td>\n",
       "      <td>D</td>\n",
       "      <td>O</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2025-01-23</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>CHL</td>\n",
       "      <td>CHILE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>96032900</td>\n",
       "      <td>BROCHAS DE AFEITAR, CEPILLOS P.CABELLO, PESTAN...</td>\n",
       "      <td>ESCOBILLAMIDWEST AEROSPACE, LTD-FDE PLASTICO, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>383012 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       channelno            dataid iesign datatype writeoffflag  \\\n",
       "0           zdzj  44b5d0c1476c46be      I        D            O   \n",
       "1           zdzj  f863182cba6b4bcc      I        D            O   \n",
       "2           zdzj  9e14d8c8f06b4c86      I        D            O   \n",
       "3           zdzj  fda26d8758334918      I        D            O   \n",
       "4           zdzj  cbae64933a2f4433      I        D            O   \n",
       "...          ...               ...    ...      ...          ...   \n",
       "383007      zdzj  e340a52adeec46f7      I        D            O   \n",
       "383008      zdzj  7329b1b93d874345      I        D            O   \n",
       "383009      zdzj  12d1d3b488f842ed      I        D            O   \n",
       "383010      zdzj  360d2f62531348b9      I        D            O   \n",
       "383011      zdzj  18d98b795b844c89      I        D            O   \n",
       "\n",
       "        writeoffdataid  outputdate origincountrycode  \\\n",
       "0                  NaN  2025-01-08               CHN   \n",
       "1                  NaN  2025-01-09               USA   \n",
       "2                  NaN  2025-01-10               USA   \n",
       "3                  NaN  2025-01-22               ESP   \n",
       "4                  NaN  2025-01-20               USA   \n",
       "...                ...         ...               ...   \n",
       "383007             NaN  2025-01-20               USA   \n",
       "383008             NaN  2025-01-20               USA   \n",
       "383009             NaN  2025-01-20               USA   \n",
       "383010             NaN  2025-01-23               USA   \n",
       "383011             NaN  2025-01-23               USA   \n",
       "\n",
       "                   origincountry countrycodeofdelivery countryofdelivery  \\\n",
       "0                          CHINA                   CHL             CHILE   \n",
       "1       UNITED STATES OF AMERICA                   CHL             CHILE   \n",
       "2       UNITED STATES OF AMERICA                   CHL             CHILE   \n",
       "3                          SPAIN                   CHL             CHILE   \n",
       "4       UNITED STATES OF AMERICA                   CHL             CHILE   \n",
       "...                          ...                   ...               ...   \n",
       "383007  UNITED STATES OF AMERICA                   CHL             CHILE   \n",
       "383008  UNITED STATES OF AMERICA                   CHL             CHILE   \n",
       "383009  UNITED STATES OF AMERICA                   CHL             CHILE   \n",
       "383010  UNITED STATES OF AMERICA                   CHL             CHILE   \n",
       "383011  UNITED STATES OF AMERICA                   CHL             CHILE   \n",
       "\n",
       "       importername  importeraddress  importercontact  suppliername  \\\n",
       "0               NaN              NaN              NaN           NaN   \n",
       "1               NaN              NaN              NaN           NaN   \n",
       "2               NaN              NaN              NaN           NaN   \n",
       "3               NaN              NaN              NaN           NaN   \n",
       "4               NaN              NaN              NaN           NaN   \n",
       "...             ...              ...              ...           ...   \n",
       "383007          NaN              NaN              NaN           NaN   \n",
       "383008          NaN              NaN              NaN           NaN   \n",
       "383009          NaN              NaN              NaN           NaN   \n",
       "383010          NaN              NaN              NaN           NaN   \n",
       "383011          NaN              NaN              NaN           NaN   \n",
       "\n",
       "        supplieraddress  suppliercontact    hscode  \\\n",
       "0                   NaN              NaN  84705000   \n",
       "1                   NaN              NaN  85176900   \n",
       "2                   NaN              NaN  71082000   \n",
       "3                   NaN              NaN  84715000   \n",
       "4                   NaN              NaN  95030099   \n",
       "...                 ...              ...       ...   \n",
       "383007              NaN              NaN  90262090   \n",
       "383008              NaN              NaN  32091019   \n",
       "383009              NaN              NaN  84091000   \n",
       "383010              NaN              NaN  73262000   \n",
       "383011              NaN              NaN  96032900   \n",
       "\n",
       "                                        hscodedescription  \\\n",
       "0                                    CAJAS REGISTRADORAS.   \n",
       "1       LOS DEMAS APARATOS PARA EMISION, TRANSMISION O...   \n",
       "2                                 ORO PARA USO MONETARIO.   \n",
       "3       UNIDADES DE PROCESO DIGITALES, EXCEPTO DE LAS ...   \n",
       "4                                              LOS DEMÃS   \n",
       "...                                                   ...   \n",
       "383007  LOS DEMAS INSTRUMENTOS PARA MEDIDA O CONTROL D...   \n",
       "383008                                          LAS DEMAS   \n",
       "383009  PARTES IDENTIFICABLES COMO DESTINADAS EXCLUSIV...   \n",
       "383010      MANUFACTURAS DE ALAMBRE DE HIERRO O DE ACERO.   \n",
       "383011  BROCHAS DE AFEITAR, CEPILLOS P.CABELLO, PESTAN...   \n",
       "\n",
       "                                     commoditydescription  \n",
       "0       TERMINAL DE PAGOWISEASY-FN5APARATO ELECTRONICO...  \n",
       "1                  RECEPTOR DE SENALSHUREULXD4INALAMBRICO  \n",
       "2       MONEDA DE OROAPMEX-FICONO BEETHOVENDE 1 OZ - D...  \n",
       "3       COMPUTADORLENOVOTHINKSTATIONP8MAQUINA AUTOMATI...  \n",
       "4       FIGURAS DE ACCIONBOLT ACTIONSOLDADOS INFANTERI...  \n",
       "...                                                   ...  \n",
       "383007  EQUIPOAR-TECH INTERNATIONAL, INC-FDE PRUEBA, M...  \n",
       "383008  PINTURAJAGUAR-FANTICORROSIVOMOD. TT-P-1757YGLP...  \n",
       "383009  CIGUENALAIRPARTS COMPANY INC-FDE ACERO, MOD. 1...  \n",
       "383010  DISCOSMIDWEST AEROSPACE, LTD-FDE SUJECION, MOD...  \n",
       "383011  ESCOBILLAMIDWEST AEROSPACE, LTD-FDE PLASTICO, ...  \n",
       "\n",
       "[383012 rows x 20 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('dataset/data_20250724/20251t3_csv_correct3/智利/IMPORT/202501-IMP.csv')\n",
    "# df = pd.read_csv('dataset/data_20250724/20251t3_csv_correct3/菲律宾/IMPORT/PH_IMPORT_202501.csv')\n",
    "# df = pd.read_csv('dataset/data_20250724/20251t3_csv_correct3/纳米比亚/IMPORT/202501-IMP-FULL.csv')\n",
    "# df = pd.read_csv('dataset/data_20250724/20251t3_csv_save2/纳米比亚/IMPORT/202501-IMP-FULL.csv')\n",
    "\n",
    "# df = pd.read_csv('dataset/data_20250724/20251t3_csv_correct3/科特迪瓦/IMPORT/KT_IMPORT_202501.csv')\n",
    "# df = pd.read_csv('dataset/data_20250724/20251t3_csv_correct3/印度/IMPORT/INDIA_IMPORT_202501_2105A.csv')\n",
    "# df = pd.read_csv('dataset/data_20250724/20251t3_csv_correct3/印度/EXPORT/INDIA_EXPORT_202501_2106.csv')\n",
    "\n",
    "df.iloc[:, 0:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e813144f-e35c-4234-9bc0-b1606a1405fa",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>totalcifvalue</th>\n",
       "      <th>totalfobvalue</th>\n",
       "      <th>grossweight</th>\n",
       "      <th>netweight</th>\n",
       "      <th>quantity</th>\n",
       "      <th>quantityunit</th>\n",
       "      <th>teu</th>\n",
       "      <th>importer_forwarderagent</th>\n",
       "      <th>supplier_forwarderagent</th>\n",
       "      <th>abnormaldata</th>\n",
       "      <th>portofloading</th>\n",
       "      <th>portofdestination</th>\n",
       "      <th>loadingcountrycode</th>\n",
       "      <th>loadingcountry</th>\n",
       "      <th>transportterm</th>\n",
       "      <th>tradeterm</th>\n",
       "      <th>paymentterm</th>\n",
       "      <th>carrier</th>\n",
       "      <th>containerno</th>\n",
       "      <th>vesselname</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24330.00</td>\n",
       "      <td>22026.910000</td>\n",
       "      <td>233.70</td>\n",
       "      <td>NaN</td>\n",
       "      <td>310.0000</td>\n",
       "      <td>UNIDADES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>HONG KONG</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>CHN</td>\n",
       "      <td>CHINA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CIF</td>\n",
       "      <td>PAGO ANTICIPADO A FECHA DE EMBARQUE</td>\n",
       "      <td>UPS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3329.20</td>\n",
       "      <td>3210.000000</td>\n",
       "      <td>10.82</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.0000</td>\n",
       "      <td>UNIDADES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>MIAMI</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CYS</td>\n",
       "      <td>PAGO ANTICIPADO A FECHA DE EMBARQUE</td>\n",
       "      <td>DHL WORLDWIDE EXP. C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3173.59</td>\n",
       "      <td>3106.460000</td>\n",
       "      <td>0.35</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2692</td>\n",
       "      <td>KILOGRAMOS NETOS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>MIAMI</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>SWE</td>\n",
       "      <td>SWEDEN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FOB</td>\n",
       "      <td>PAGO ANTICIPADO A FECHA DE EMBARQUE</td>\n",
       "      <td>DHL WORLDWIDE EXP. C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7274.26</td>\n",
       "      <td>7023.780000</td>\n",
       "      <td>21.88</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>UNIDADES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>OTROS PTOS.ESPANA</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>ESP</td>\n",
       "      <td>SPAIN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FOB</td>\n",
       "      <td>PAGO ANTICIPADO A FECHA DE EMBARQUE</td>\n",
       "      <td>AVIANCA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>170.74</td>\n",
       "      <td>161.678309</td>\n",
       "      <td>23.62</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.7077</td>\n",
       "      <td>KILOGRAMOS NETOS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>MIAMI</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FOB</td>\n",
       "      <td>PAGO ANTICIPADO A FECHA DE EMBARQUE</td>\n",
       "      <td>DHL WORLDWIDE EXP. C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383007</th>\n",
       "      <td>926.34</td>\n",
       "      <td>901.831071</td>\n",
       "      <td>3.88</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>UNIDADES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NEW YORK</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>AIR</td>\n",
       "      <td>CYS</td>\n",
       "      <td>COBRANZA HASTA 1 AÃ‘O</td>\n",
       "      <td>LATAM AIRLINES GROUP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383008</th>\n",
       "      <td>1384.00</td>\n",
       "      <td>850.000000</td>\n",
       "      <td>12.44</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.5692</td>\n",
       "      <td>KILOGRAMOS NETOS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>MIAMI</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>AIR</td>\n",
       "      <td>CYS</td>\n",
       "      <td>COBRANZA HASTA 1 AÃ‘O</td>\n",
       "      <td>ATLAS AIR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383009</th>\n",
       "      <td>17818.80</td>\n",
       "      <td>17668.130000</td>\n",
       "      <td>25.59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19.6846</td>\n",
       "      <td>KILOGRAMOS NETOS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>MIAMI</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>AIR</td>\n",
       "      <td>CYS</td>\n",
       "      <td>COBRANZA HASTA 1 AÃ‘O</td>\n",
       "      <td>KOREAN AIR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383010</th>\n",
       "      <td>1153.42</td>\n",
       "      <td>606.253636</td>\n",
       "      <td>80.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20.8538</td>\n",
       "      <td>KILOGRAMOS NETOS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>MIAMI</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>AIR</td>\n",
       "      <td>CYS</td>\n",
       "      <td>COBRANZA HASTA 1 AÃ‘O</td>\n",
       "      <td>ATLAS AIR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383011</th>\n",
       "      <td>384.47</td>\n",
       "      <td>202.082793</td>\n",
       "      <td>80.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0000</td>\n",
       "      <td>UNIDADES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>MIAMI</td>\n",
       "      <td>AEROPUERTO COM. A. MERINO B.</td>\n",
       "      <td>USA</td>\n",
       "      <td>UNITED STATES OF AMERICA</td>\n",
       "      <td>AIR</td>\n",
       "      <td>CYS</td>\n",
       "      <td>COBRANZA HASTA 1 AÃ‘O</td>\n",
       "      <td>ATLAS AIR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>383012 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        totalcifvalue  totalfobvalue  grossweight  netweight  quantity  \\\n",
       "0            24330.00   22026.910000       233.70        NaN  310.0000   \n",
       "1             3329.20    3210.000000        10.82        NaN    7.0000   \n",
       "2             3173.59    3106.460000         0.35        NaN    0.2692   \n",
       "3             7274.26    7023.780000        21.88        NaN    1.0000   \n",
       "4              170.74     161.678309        23.62        NaN    0.7077   \n",
       "...               ...            ...          ...        ...       ...   \n",
       "383007         926.34     901.831071         3.88        NaN    1.0000   \n",
       "383008        1384.00     850.000000        12.44        NaN    9.5692   \n",
       "383009       17818.80   17668.130000        25.59        NaN   19.6846   \n",
       "383010        1153.42     606.253636        80.00        NaN   20.8538   \n",
       "383011         384.47     202.082793        80.00        NaN    2.0000   \n",
       "\n",
       "            quantityunit  teu  importer_forwarderagent  \\\n",
       "0               UNIDADES  NaN                        0   \n",
       "1               UNIDADES  NaN                        0   \n",
       "2       KILOGRAMOS NETOS  NaN                        0   \n",
       "3               UNIDADES  NaN                        0   \n",
       "4       KILOGRAMOS NETOS  NaN                        0   \n",
       "...                  ...  ...                      ...   \n",
       "383007          UNIDADES  NaN                        0   \n",
       "383008  KILOGRAMOS NETOS  NaN                        0   \n",
       "383009  KILOGRAMOS NETOS  NaN                        0   \n",
       "383010  KILOGRAMOS NETOS  NaN                        0   \n",
       "383011          UNIDADES  NaN                        0   \n",
       "\n",
       "        supplier_forwarderagent  abnormaldata      portofloading  \\\n",
       "0                             0             0          HONG KONG   \n",
       "1                             0             0              MIAMI   \n",
       "2                             0             0              MIAMI   \n",
       "3                             0             0  OTROS PTOS.ESPANA   \n",
       "4                             0             0              MIAMI   \n",
       "...                         ...           ...                ...   \n",
       "383007                        0             0           NEW YORK   \n",
       "383008                        0             0              MIAMI   \n",
       "383009                        0             0              MIAMI   \n",
       "383010                        0             0              MIAMI   \n",
       "383011                        0             0              MIAMI   \n",
       "\n",
       "                   portofdestination loadingcountrycode  \\\n",
       "0       AEROPUERTO COM. A. MERINO B.                CHN   \n",
       "1       AEROPUERTO COM. A. MERINO B.                USA   \n",
       "2       AEROPUERTO COM. A. MERINO B.                SWE   \n",
       "3       AEROPUERTO COM. A. MERINO B.                ESP   \n",
       "4       AEROPUERTO COM. A. MERINO B.                USA   \n",
       "...                              ...                ...   \n",
       "383007  AEROPUERTO COM. A. MERINO B.                USA   \n",
       "383008  AEROPUERTO COM. A. MERINO B.                USA   \n",
       "383009  AEROPUERTO COM. A. MERINO B.                USA   \n",
       "383010  AEROPUERTO COM. A. MERINO B.                USA   \n",
       "383011  AEROPUERTO COM. A. MERINO B.                USA   \n",
       "\n",
       "                  loadingcountry transportterm tradeterm  \\\n",
       "0                          CHINA           NaN       CIF   \n",
       "1       UNITED STATES OF AMERICA           NaN       CYS   \n",
       "2                         SWEDEN           NaN       FOB   \n",
       "3                          SPAIN           NaN       FOB   \n",
       "4       UNITED STATES OF AMERICA           NaN       FOB   \n",
       "...                          ...           ...       ...   \n",
       "383007  UNITED STATES OF AMERICA           AIR       CYS   \n",
       "383008  UNITED STATES OF AMERICA           AIR       CYS   \n",
       "383009  UNITED STATES OF AMERICA           AIR       CYS   \n",
       "383010  UNITED STATES OF AMERICA           AIR       CYS   \n",
       "383011  UNITED STATES OF AMERICA           AIR       CYS   \n",
       "\n",
       "                                paymentterm               carrier  \\\n",
       "0       PAGO ANTICIPADO A FECHA DE EMBARQUE                   UPS   \n",
       "1       PAGO ANTICIPADO A FECHA DE EMBARQUE  DHL WORLDWIDE EXP. C   \n",
       "2       PAGO ANTICIPADO A FECHA DE EMBARQUE  DHL WORLDWIDE EXP. C   \n",
       "3       PAGO ANTICIPADO A FECHA DE EMBARQUE               AVIANCA   \n",
       "4       PAGO ANTICIPADO A FECHA DE EMBARQUE  DHL WORLDWIDE EXP. C   \n",
       "...                                     ...                   ...   \n",
       "383007                COBRANZA HASTA 1 AÃ‘O  LATAM AIRLINES GROUP   \n",
       "383008                COBRANZA HASTA 1 AÃ‘O             ATLAS AIR   \n",
       "383009                COBRANZA HASTA 1 AÃ‘O            KOREAN AIR   \n",
       "383010                COBRANZA HASTA 1 AÃ‘O             ATLAS AIR   \n",
       "383011                COBRANZA HASTA 1 AÃ‘O             ATLAS AIR   \n",
       "\n",
       "        containerno  vesselname  \n",
       "0               NaN         NaN  \n",
       "1               NaN         NaN  \n",
       "2               NaN         NaN  \n",
       "3               NaN         NaN  \n",
       "4               NaN         NaN  \n",
       "...             ...         ...  \n",
       "383007          NaN         NaN  \n",
       "383008          NaN         NaN  \n",
       "383009          NaN         NaN  \n",
       "383010          NaN         NaN  \n",
       "383011          NaN         NaN  \n",
       "\n",
       "[383012 rows x 20 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[:, 20:40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "7956240e-5da5-446a-b34d-7e50eef33f75",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>brand</th>\n",
       "      <th>version</th>\n",
       "      <th>country</th>\n",
       "      <th>IMPORTER_ID</th>\n",
       "      <th>SUPPLIER_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383007</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383008</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383009</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383010</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383011</th>\n",
       "      <td>NaN</td>\n",
       "      <td>202501</td>\n",
       "      <td>CHL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>383012 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       brand  version country  IMPORTER_ID  SUPPLIER_ID\n",
       "0        NaN   202501     CHL          NaN          NaN\n",
       "1        NaN   202501     CHL          NaN          NaN\n",
       "2        NaN   202501     CHL          NaN          NaN\n",
       "3        NaN   202501     CHL          NaN          NaN\n",
       "4        NaN   202501     CHL          NaN          NaN\n",
       "...      ...      ...     ...          ...          ...\n",
       "383007   NaN   202501     CHL          NaN          NaN\n",
       "383008   NaN   202501     CHL          NaN          NaN\n",
       "383009   NaN   202501     CHL          NaN          NaN\n",
       "383010   NaN   202501     CHL          NaN          NaN\n",
       "383011   NaN   202501     CHL          NaN          NaN\n",
       "\n",
       "[383012 rows x 5 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[:, 40:60]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51d9eaf7-cc19-445d-b8c3-30cf67837953",
   "metadata": {},
   "source": [
    "# 8 特殊处理-坦桑尼亚/乌克兰/智利"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdb1752c-626e-491f-85ea-3140b7ab6d99",
   "metadata": {},
   "source": [
    "## 8.1 坦桑尼亚5月份数据修正"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "f5e16368-d807-4693-bc12-a637678f0ad0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cols = ['TRANSPORT_MODE_CODE', 'TRANSPORT_MODE_NAME']\n",
    "# tmp = df1[cols].fillna('').astype(str).values\n",
    "# unique = [list(t) for t in {tuple(row) for row in tmp}]\n",
    "# print(unique)\n",
    "\n",
    "# mapping_dict = dict()\n",
    "# for x in unique:\n",
    "#     mapping_dict[x[0]]=x[1]\n",
    "# mapping_dict['']='UNKNOWN'\n",
    "# mapping_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b0038b42-f86e-4004-a265-ed5be1542cb2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mapping_dict = {'7.0': 'Transport on fixed installation',\n",
    " '3.0': 'Road transport',\n",
    " '9.0': 'UNKNOWN',\n",
    " '': 'UNKNOWN',\n",
    " 'nan': 'UNKNOWN',\n",
    " '1.0': 'Sea transport',\n",
    " '5.0': 'Postal Transport',\n",
    " '4.0': 'Air transport',\n",
    " '8.0': 'Inland waterways transport',\n",
    " '2.0': 'Rail transport',\n",
    " '6.0': 'Multimodal Transport',\n",
    " '7': 'Transport on fixed installation',\n",
    " '3': 'Road transport',\n",
    " '9': 'UNKNOWN',\n",
    " '1': 'Sea transport',\n",
    " '5': 'Postal Transport',\n",
    " '4': 'Air transport',\n",
    " '8': 'Inland waterways transport',\n",
    " '2': 'Rail transport',\n",
    " '6': 'Multimodal Transport'\n",
    "               }\n",
    "\n",
    "# mapping_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a53d6e7-0077-4bbc-935a-782fdd14438e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "71e0f109-a752-43a2-a0fb-606c3ebce482",
   "metadata": {},
   "outputs": [],
   "source": [
    "pth1 = Path('dataset/data_20250724/20254t6_src_version/坦桑尼亚/import_paths/202505-IMP-RAW.csv')\n",
    "pth2 = Path('dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202505-IMP-RAW.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "aa06c92f-c89a-45a9-9380-64b82dd34778",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df1 = pd.read_csv(pth1, dtype='str')\n",
    "tmp1 = df1.loc[:, ['dataid', 'TRANSPORT_MODE_CODE']]\n",
    "tmp1['TRANSPORT_MODE_CODE'] = df1['TRANSPORT_MODE_CODE'].astype('str').apply(lambda x: mapping_dict[x])\n",
    "# tmp1\n",
    "\n",
    "df2 = pd.read_csv(pth2, dtype='str')\n",
    "target_cols = df2.columns.values\n",
    "# df2\n",
    "\n",
    "df_tmp2 = pd.merge(df2, tmp1, on='dataid', how='left')\n",
    "\n",
    "df_tmp2.drop(columns=['transportterm'], inplace=True)\n",
    "df_tmp2.rename(columns={'TRANSPORT_MODE_CODE':'transportterm'}, inplace=True)\n",
    "df_tmp2 = df_tmp2.loc[:, df2.columns.values]\n",
    "# df_tmp2\n",
    "\n",
    "df_tmp2.loc[:, target_cols].to_csv(pth2\n",
    "                                   , sep=','\n",
    "                                   , index=False\n",
    "                                   , quotechar='\"'         # 双引号导引符\n",
    "                                   , quoting=csv.QUOTE_ALL        # 强制所有字段加引号\n",
    "                               , lineterminator='\\n', header=True, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "7462afb7-947b-4523-aa9f-7126b89950a7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# df_tmp2.loc[:,['dataid', 'transportterm']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2c6dfe75-5a9b-4357-91cb-2d00c1c4da6d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pth1 = Path('dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202505-EXP-RAW.csv')\n",
    "pth2 = Path('dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-EXP-RAW.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d0eb35c2-f4bf-432f-adac-ef611057e0fd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df1 = pd.read_csv(pth1, dtype='str')\n",
    "tmp1 = df1.loc[:, ['dataid', 'TRANSPORT_MODE_CODE']]\n",
    "# tmp1['TRANSPORT_MODE_CODE'] = df1['TRANSPORT_MODE_CODE'].astype('int').apply('{:.1f}'.format).apply(lambda x: mapping_dict[x])\n",
    "tmp1['TRANSPORT_MODE_CODE'] = df1['TRANSPORT_MODE_CODE'].astype('str').apply(lambda x: mapping_dict[x])\n",
    "# tmp1\n",
    "\n",
    "df2 = pd.read_csv(pth2, dtype='str')\n",
    "target_cols = df2.columns.values\n",
    "# df2\n",
    "\n",
    "df_tmp2 = pd.merge(df2, tmp1, on='dataid', how='left')\n",
    "\n",
    "df_tmp2.drop(columns=['transportterm'], inplace=True)\n",
    "df_tmp2.rename(columns={'TRANSPORT_MODE_CODE':'transportterm'}, inplace=True)\n",
    "df_tmp2 = df_tmp2.loc[:, df2.columns.values]\n",
    "# df_tmp2\n",
    "\n",
    "df_tmp2.loc[:, target_cols].to_csv(pth2\n",
    "                                   , sep=','\n",
    "                                   , index=False\n",
    "                                   , quotechar='\"'         # 双引号导引符\n",
    "                                   , quoting=csv.QUOTE_ALL        # 强制所有字段加引号\n",
    "                               , lineterminator='\\n', header=True, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "ba9051cf-e213-4c70-9db8-8e7a79dd052f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_tmp2.loc[:,['dataid', 'transportterm']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "12d3435a-b4cb-4e77-8293-6704fff753bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "pth1 = Path('dataset/data_20250724/20254t6_src_version/坦桑尼亚/export_paths/202505-TRANS-RAW.csv')\n",
    "pth2 = Path('dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "0231d67e-24f5-43ab-b6b4-1e55835f52c5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df1 = pd.read_csv(pth1, dtype='str')\n",
    "tmp1 = df1.loc[:, ['dataid', 'TRANSPORT_MODE_CODE']]\n",
    "tmp1['TRANSPORT_MODE_CODE'] = df1['TRANSPORT_MODE_CODE'].astype('str').apply(lambda x: mapping_dict[x])\n",
    "# tmp1\n",
    "\n",
    "df2 = pd.read_csv(pth2, dtype='str')\n",
    "target_cols = df2.columns.values\n",
    "# df2\n",
    "\n",
    "df_tmp2 = pd.merge(df2, tmp1, on='dataid', how='left')\n",
    "\n",
    "df_tmp2.drop(columns=['transportterm'], inplace=True)\n",
    "df_tmp2.rename(columns={'TRANSPORT_MODE_CODE':'transportterm'}, inplace=True)\n",
    "df_tmp2 = df_tmp2.loc[:, df2.columns.values]\n",
    "# df_tmp2\n",
    "\n",
    "df_tmp2.loc[:, target_cols].to_csv(pth2\n",
    "                                   , sep=','\n",
    "                                   , index=False\n",
    "                                   , quotechar='\"'         # 双引号导引符\n",
    "                                   , quoting=csv.QUOTE_ALL        # 强制所有字段加引号\n",
    "                               , lineterminator='\\n', header=True, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "9abcb914-7b71-41d0-b3fb-9c62613f61b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataid</th>\n",
       "      <th>transportterm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>76d4ceb482744909</td>\n",
       "      <td>Air transport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>33ad5e5675b546f7</td>\n",
       "      <td>Air transport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bd2b65d24fce4b45</td>\n",
       "      <td>Air transport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5b5924f39ffe4a8e</td>\n",
       "      <td>Air transport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ff8a0dc4cca7462d</td>\n",
       "      <td>Air transport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125610</th>\n",
       "      <td>e93ddf58bcae4975</td>\n",
       "      <td>Sea transport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125611</th>\n",
       "      <td>b474907975604550</td>\n",
       "      <td>Sea transport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125612</th>\n",
       "      <td>288029c11da94b1b</td>\n",
       "      <td>Sea transport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125613</th>\n",
       "      <td>354e6075f4184396</td>\n",
       "      <td>Sea transport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125614</th>\n",
       "      <td>7a2292e9d56b4f67</td>\n",
       "      <td>Air transport</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125615 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  dataid  transportterm\n",
       "0       76d4ceb482744909  Air transport\n",
       "1       33ad5e5675b546f7  Air transport\n",
       "2       bd2b65d24fce4b45  Air transport\n",
       "3       5b5924f39ffe4a8e  Air transport\n",
       "4       ff8a0dc4cca7462d  Air transport\n",
       "...                  ...            ...\n",
       "125610  e93ddf58bcae4975  Sea transport\n",
       "125611  b474907975604550  Sea transport\n",
       "125612  288029c11da94b1b  Sea transport\n",
       "125613  354e6075f4184396  Sea transport\n",
       "125614  7a2292e9d56b4f67  Air transport\n",
       "\n",
       "[125615 rows x 2 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tmp2.loc[:,['dataid', 'transportterm']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0ca1202-4ce5-40fb-a290-0962d0fb8fba",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "120e1dca-1794-4755-93fb-54e86a836561",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "563f7102-f356-449b-b584-612237ec9253",
   "metadata": {},
   "source": [
    "## 8.2 '智利', '坦桑尼亚', '乌克兰' transportterm统一修正"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "a84bea2f-8e0a-49b9-9875-dd51c6e74d51",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transtype_mapping_dict = {\n",
    "#     'AEREO': 'AIR',  # 西班牙语\"航空\"，匹配 AIR（航空运输）\n",
    "#     'AIR TRANSPORT': 'AIR',  # 英语\"航空运输\"，直接匹配 AIR\n",
    "#     'CARRETERO': 'ROAD',  # 西班牙语\"公路\"，匹配 ROAD（公路运输）\n",
    "#     'INLAND WATERWAYS TRANSPORT': 'INLAND WATER',  # 英语\"内河运输\"，直接匹配 INLAND WATER（内河运输）\n",
    "#     'MARITIMA': 'SEA',  # 西班牙语\"海洋\"，匹配 SEA（海洋运输）\n",
    "#     'MULTIMODAL TRANSPORT': 'MULTI-MODAL',  # 英语\"多式联运\"，直接匹配 MULTI-MODAL（多式联运）\n",
    "#     'OLEODUCTOS': 'PIPELINE',  # 西班牙语\"管道\"，匹配 PIPELINE（管道运输）\n",
    "#     'OTRO': 'OTHERS',  # 西班牙语\"其他\"，匹配 OTHERS（其他）\n",
    "#     'OTROS': 'OTHERS',  # 西班牙语\"其他\"（复数），匹配 OTHERS\n",
    "#     'POSTAL TRANSPORT': 'MAIL',  # 英语\"邮政运输\"，直接匹配 MAIL（邮政运输）\n",
    "#     'RAIL TRANSPORT': 'RAILWAY',  # 英语\"铁路运输\"，直接匹配 RAILWAY（铁路运输）\n",
    "#     'ROAD TRANSPORT': 'ROAD',  # 英语\"公路运输\"，直接匹配 ROAD\n",
    "#     'SEA TRANSPORT': 'SEA',  # 英语\"海洋运输\"，直接匹配 SEA\n",
    "#     'TENDIDO ELECTRICO (AEREO,SUBT)': 'OTHERS',  # 西班牙语\"电力线路（空中、地下）\"，不属于标准货物运输方式，归入 OTHERS\n",
    "#     'TRANSPORT ON FIXED INSTALLATION': 'PIPELINE',  # 英语\"固定设施上的运输\"，部分匹配 PIPELINE（管道是固定设施的一种）\n",
    "#     'UNKNOWN': 'OTHERS',  # 英语\"未知\"，匹配 OTHERS\n",
    "#     'АВТОТРАНСПОРТ СВОЇМ ХОДОМ ЯК ТОВАР': 'ROAD',  # 乌克兰语\"作为货物运输的汽车（用自己的动力）\"，匹配 ROAD（公路运输）\n",
    "#     'ВАНТАЖНИЙ АВТОМОБІЛЬ': 'ROAD',  # 乌克兰语\"卡车\"，匹配 ROAD\n",
    "#     'ВАНТАЖНИЙ АВТОМОБІЛЬ НА МОРСЬКОМУ СУДНІ': 'SEA',  # 乌克兰语\"在船上的卡车\"，运输方式为海运，匹配 SEA\n",
    "#     'ЗАЛІЗНИЧНИЙ ВАГОН': 'RAILWAY',  # 乌克兰语\"铁路车厢\"，匹配 RAILWAY\n",
    "#     'КОНТЕЙНЕР НА ВАНТАЖНОМУ АВТОМОБІЛІ': 'ROAD',  # 乌克兰语\"在卡车上的集装箱\"，运输方式为公路，匹配 ROAD\n",
    "#     'КОНТЕЙНЕР НА ЗАЛІЗНИЧНОМУ ВАГОНІ': 'RAILWAY',  # 乌克兰语\"在铁路车厢上的集装箱\"，匹配 RAILWAY\n",
    "#     'КОНТЕЙНЕР НА МОРСЬКОМУ СУДНІ': 'SEA',  # 乌克兰语\"在船上的集装箱\"，匹配 SEA\n",
    "#     'МОРСЬКЕ СУДНО': 'SEA',  # 乌克兰语\"海船\"，匹配 SEA\n",
    "#     'НЕВІДОМИЙ': 'OTHERS',  # 乌克兰语\"未知\"，匹配 OTHERS\n",
    "#     'ПОШТОВЕ ВІДПРАВЛЕННЯ АВТОТРАНСПОРТОМ': 'MAIL',  # 乌克兰语\"汽车运输的邮政发送\"，运输类型为邮政，匹配 MAIL\n",
    "#     'ТРУБОПРОВІДНИЙ ТРАНСПОРТ': 'PIPELINE'  # 乌克兰语\"管道运输\"，直接匹配 PIPELINE\n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "8710daeb-e139-4b1b-b4a7-b812346799fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "transtype_mapping_dict = {\n",
    "    \"VESSEL, CONTAINERIZED\": \"SEA\",\n",
    "    \"L\": \"LAND\",\n",
    "    \"NOT DECLARED\": \"OTHERS\",\n",
    "    \"PIPELINE\": \"PIPELINE\",\n",
    "    \"AERIAL\": \"AIR\",\n",
    "    \"AIR\": \"AIR\",\n",
    "    \"LAND - OTHER\": \"LAND\",\n",
    "    \"OLEODUCTOS\": \"PIPELINE\",\n",
    "    \"АВТОТРАНСПОРТ СВОЇМ ХОДОМ ЯК ТОВАР\": \"ROAD\",\n",
    "    \"ВАНТАЖНИЙ АВТОМОБІЛЬ\": \"ROAD\",\n",
    "    \"MARITIMO\": \"SEA\",\n",
    "    \"RAI\": \"RAILWAY\",\n",
    "    \"ЗАЛІЗНИЧНИЙ ВАГОН\": \"RAILWAY\",\n",
    "    \"MARITIMA\": \"SEA\",\n",
    "    \"LCS-SEA\": \"SEA\",\n",
    "    \"VESSEL, NON-CONTAINER\": \"SEA\",\n",
    "    \"TRANSPORT PAR AIR\": \"AIR\",\n",
    "    \"OTRO\": \"OTHERS\",\n",
    "    \"ROAD\": \"ROAD\",\n",
    "    \"ACUÁTICA\": \"INLAND WATER\",\n",
    "    \"MARITIME TRANSPORT\": \"SEA\",\n",
    "    \"nan\": \"OTHERS\",\n",
    "    \"AÉREA\": \"AIR\",\n",
    "    \"OTHER\": \"OTHERS\",\n",
    "    \"RAIL\": \"RAILWAY\",\n",
    "    \"COURIER (ADUANA AÉREA)\": \"AIR\",\n",
    "    \"R\": \"RAILWAY\",\n",
    "    \"VESSEL\": \"SEA\",\n",
    "    \"FERROVIARIA\": \"RAILWAY\",\n",
    "    \"ВАНТАЖНИЙ АВТОМОБІЛЬ НА МОРСЬКОМУ СУДНІ\": \"SEA\",\n",
    "    \"TENDIDO ELECTRICO (AEREO,SUBT)\": \"OTHERS\",\n",
    "    \"BY SEA\": \"SEA\",\n",
    "    \"MARÃ\\xadTIMO\": \"SEA\",\n",
    "    \"MARÍTIMO\": \"SEA\",\n",
    "    \"ПОШТОВЕ ВІДПРАВЛЕННЯ АВТОТРАНСПОРТОМ\": \"MAIL\",\n",
    "    \"ENCOMIENDA\": \"MAIL\",\n",
    "    \"INSTAL FIJAS\": \"OTHERS\",\n",
    "    \"AEREA\": \"AIR\",\n",
    "    \"LCS-ROAD\": \"ROAD\",\n",
    "    \"CONDUCTOR ELÉCTRICO\": \"OTHERS\",\n",
    "    \"AEREO\": \"AIR\",\n",
    "    \"SEA\": \"SEA\",\n",
    "    \"FLUVIAL\": \"INLAND WATER\",\n",
    "    \"MARITIME\": \"SEA\",\n",
    "    \"OWN MEANS\": \"OTHERS\",\n",
    "    \"O\": \"OTHERS\",\n",
    "    \"ТРУБОПРОВІДНИЙ ТРАНСПОРТ\": \"PIPELINE\",\n",
    "    \"ICD\": \"OTHERS\",\n",
    "    \"КОНТЕЙНЕР НА ВАНТАЖНОМУ АВТОМОБІЛІ\": \"ROAD\",\n",
    "    \"LAND - RAILWAY\": \"RAILWAY\",\n",
    "    \"TUBERIAS\": \"PIPELINE\",\n",
    "    \"TERRESTRIAL\": \"LAND\",\n",
    "    \"A\": \"AIR\",\n",
    "    \"OTROS\": \"OTHERS\",\n",
    "    \"S\": \"SEA\",\n",
    "    \"AIR TRANSPORT\": \"AIR\",\n",
    "    \"AÉREO\": \"AIR\",\n",
    "    \"ROAD TRANSPORT\": \"ROAD\",\n",
    "    \"AERIEN\": \"AIR\",\n",
    "    \"CORREO\": \"MAIL\",\n",
    "    \"AÃ©REA\": \"AIR\",\n",
    "    \"CONDUCTOR ELÃ©CTRICO\": \"OTHERS\",\n",
    "    \"TRANSPORT PAR ROUTE\": \"ROAD\",\n",
    "    \"RAIL TRANSPORT\": \"RAILWAY\",\n",
    "    \"LCS\": \"OTHERS\",\n",
    "    \"ICD-SEA\": \"SEA\",\n",
    "    \"ACUÃ¡TICA\": \"INLAND WATER\",\n",
    "    \"AÃ©REO\": \"AIR\",\n",
    "    \"МОРСЬКЕ СУДНО\": \"SEA\",\n",
    "    \"ICD-ROAD\": \"ROAD\",\n",
    "    \"TRUCK\": \"ROAD\",\n",
    "    \"TRANSPORT MARITIME\": \"SEA\",\n",
    "    \"DUCTOS\": \"PIPELINE\",\n",
    "    \"КОНТЕЙНЕР НА ЗАЛІЗНИЧНОМУ ВАГОНІ\": \"RAILWAY\",\n",
    "    \"POSTAL\": \"MAIL\",\n",
    "    \"TERRESTRE\": \"LAND\",\n",
    "    \"OTRAS VIAS\": \"OTHERS\",\n",
    "    \"CARRETERA\": \"ROAD\",\n",
    "    \"CARRETERO\": \"ROAD\",\n",
    "    \"НЕВІДОМИЙ\": \"OTHERS\",\n",
    "    \"ARREO\": \"OTHERS\",\n",
    "    \"КОНТЕЙНЕР НА МОРСЬКОМУ СУДНІ\": \"SEA\",\n",
    "    \"OTHERS\": \"OTHERS\",\n",
    "    \"UNKNOWN\": \"OTHERS\", # 未知运输方式，归类为其他\n",
    "    \"TRANSPORT ON FIXED INSTALLATION\": \"OTHERS\", # 固定设施运输，无直接对应标准类型，归类为其他\n",
    "    \"INLAND WATERWAYS TRANSPORT\": \"INLAND WATER\", # 内河运输，直接匹配 INLAND WATER（内河运输）\n",
    "    \"SEA TRANSPORT\": \"SEA\", # 海洋运输，直接匹配 SEA（海洋运输）\n",
    "    \"MULTIMODAL TRANSPORT\": \"MULTI-MODAL\", # 多式联运，直接匹配 MULTI-MODAL（多式联运）\n",
    "    \"POSTAL TRANSPORT\": \"MAIL\", # 邮政运输，直接匹配 MAIL（邮政运输）\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "d0f0994d-c2e1-4304-b618-083a58efa2e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from_dir=Path('dataset/data_20250724/20254t6_csv_correct1')\n",
    "to_dir=Path('dataset/data_20250724/20254t6_csv_correct1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "fce9a582-a16b-4247-9291-67e23a25ac9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# country_list = ['智利']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f16a527c-340e-4644-a4b7-ed9992780e72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transtype_list = get_transtype_list(country_list, from_dir, transtype_cols = ['transportterm'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "739baf5c-4763-402c-b052-2b679abc7da5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace_transtype(country_list, transtype_mapping_dict, from_dir, to_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "0a8f633a-8bb6-4473-9c62-d59120941b7d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:55:05,368 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:55:05,369 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 06:55:05,369 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:55:08,829 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 06:55:08,836 - File Process - INFO - transtype mapping done, the data shape is (87003, 45)\n",
      "2025-09-10 06:55:08,840 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 06:55:10,495 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:55:10,498 - File Process - INFO - transtype mapping done, the data shape is (128934, 45)\n",
      "2025-09-10 06:55:10,503 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 06:55:10,708 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 06:55:10,710 - File Process - INFO - transtype mapping done, the data shape is (6534, 45)\n",
      "2025-09-10 06:55:10,711 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 06:55:10,868 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 06:55:10,873 - File Process - INFO - transtype mapping done, the data shape is (5331, 45)\n",
      "2025-09-10 06:55:10,874 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:55:11,382 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 06:55:11,388 - File Process - INFO - transtype mapping done, the data shape is (91874, 45)\n",
      "2025-09-10 06:55:11,422 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 06:55:15,395 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 06:55:15,396 - File Process - INFO - transtype mapping done, the data shape is (112769, 45)\n",
      "2025-09-10 06:55:15,396 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 06:55:15,581 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:55:15,581 - File Process - INFO - transtype mapping done, the data shape is (138793, 45)\n",
      "2025-09-10 06:55:15,582 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 06:55:15,744 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 06:55:15,771 - File Process - INFO - transtype mapping done, the data shape is (5108, 45)\n",
      "2025-09-10 06:55:15,771 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:55:16,021 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:55:16,022 - File Process - INFO - transtype mapping done, the data shape is (134581, 45)\n",
      "2025-09-10 06:55:16,022 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:55:21,334 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:55:21,336 - File Process - INFO - transtype mapping done, the data shape is (125615, 45)\n",
      "2025-09-10 06:55:21,338 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 06:55:27,658 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:55:27,660 - File Process - INFO - transtype mapping done, the data shape is (135921, 45)\n",
      "2025-09-10 06:55:27,661 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:55:29,647 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 06:55:29,650 - File Process - INFO - transtype mapping done, the data shape is (390891, 45)\n",
      "2025-09-10 06:55:29,652 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 06:55:31,746 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 06:55:31,787 - File Process - INFO - transtype mapping done, the data shape is (387273, 45)\n",
      "2025-09-10 06:55:31,798 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 06:55:38,982 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:55:38,988 - File Process - INFO - transtype mapping done, the data shape is (147189, 45)\n",
      "2025-09-10 06:55:38,991 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:55:41,735 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 06:55:41,735 - File Process - INFO - transtype mapping done, the data shape is (344016, 45)\n",
      "2025-09-10 06:55:41,735 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 06:55:42,972 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 06:55:42,973 - File Process - INFO - transtype mapping done, the data shape is (404234, 45)\n",
      "2025-09-10 06:55:42,975 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 06:55:53,361 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 06:55:53,363 - File Process - INFO - transtype mapping done, the data shape is (310571, 45)\n",
      "2025-09-10 06:55:53,367 - File Process - INFO - transtype mapping takes 48 seconds\n",
      "2025-09-10 06:55:53,401 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:55:56,738 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 06:55:56,739 - File Process - INFO - transtype mapping done, the data shape is (400856, 45)\n",
      "2025-09-10 06:55:56,739 - File Process - INFO - transtype mapping takes 51 seconds\n",
      "2025-09-10 06:55:56,788 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:56:05,768 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:56:05,768 - File Process - INFO - transtype mapping done, the data shape is (739143, 45)\n",
      "2025-09-10 06:56:05,769 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 06:56:22,810 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 06:56:22,810 - File Process - INFO - transtype mapping done, the data shape is (744475, 45)\n",
      "2025-09-10 06:56:22,810 - File Process - INFO - the path : dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 06:56:39,053 - File Process - INFO - save path: dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 06:56:39,053 - File Process - INFO - transtype mapping done, the data shape is (755738, 45)\n",
      "2025-09-10 06:56:39,054 - File Process - INFO - transtype mapping takes 94 seconds\n",
      "2025-09-10 06:56:39,146 - Data Goverance - INFO - 1\n",
      "2025-09-10 06:56:39,151 - Data Goverance - INFO - replace_currency_to_usd takes 1.563058606783549 minutes\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "params_list = [([ctry], transtype_mapping_dict, from_dir, to_dir) for ctry in country_list]\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: replace_transtype(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        logger.info(future.result())\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'replace_currency_to_usd takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "eabb8d19-cc91-4a65-a9de-53a97e27d2ec",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"channelno\",\"dataid\",\"iesign\",\"datatype\",\"writeoffflag\",\"writeoffdataid\",\"outputdate\",\"origincountrycode\",\"origincountry\",\"countrycodeofdelivery\",\"countryofdelivery\",\"importername\",\"importeraddress\",\"importercontact\",\"suppliername\",\"supplieraddress\",\"suppliercontact\",\"hscode\",\"hscodedescription\",\"commoditydescription\",\"totalcifvalue\",\"totalfobvalue\",\"grossweight\",\"netweight\",\"quantity\",\"quantityunit\",\"teu\",\"importer_forwarderagent\",\"supplier_forwarderagent\",\"abnormaldata\",\"portofloading\",\"portofdestination\",\"loadingcountrycode\",\"loadingcountry\",\"transportterm\",\"tradeterm\",\"paymentterm\",\"carrier\",\"containerno\",\"vesselname\",\"brand\",\"version\",\"country\",\"IMPORTER_ID\",\"SUPPLIER_ID\"\n",
      "\"zdzj\",\"9aa29882257247ee\",\"I\",\"D\",\"O\",\"\",\"2025-04-24\",\"JPN\",\"JAPAN\",\"ZMB\",\"ZAMBIA\",\"JANE CHISANI\",\"\",\"\",\"BE FORWARD CO.LTD\",\"\",\"\",\"870323910000\",\"Other:Other: of cylinder capacity exceeding 1500cc but not exceeding 2000cc.\",\"USED MERCEDES BENZ C-CLASS\",\"6162.2984964895195\",\"\",\"1490.0\",\"1490.0\",\"1.0\",\"NMB\",\"\",\"0\",\"0\",\"0\",\"\",\"\",\"JPN\",\"JAPAN\",\"SEA\",\"\",\"\",\"\",\"\",\"\",\"\",\"202504\",\"TZA\",\"\",\"\"\n",
      "\"zdzj\",\"1835c73ea8a040cf\",\"I\",\"D\",\"O\",\"\",\"2025-04-24\",\"JPN\",\"JAPAN\",\"TZA\",\"TANZANIA\",\"HANSPAUL AUTOMECHS LIMITED\",\"\",\"\",\"AMADA (UK) JHB \",\"\",\"\",\"846694000000\",\"For machines of heading 84.62 or 84.63:Parts and accessories suitable for\",\"A74861  86  AFH SINGLE V DIE (V=12)\",\"2920.400074805112\",\"\",\"40.0\",\"40.0\",\"40.0\",\"KGM\",\"\",\"0\",\"0\",\"0\",\"\",\"\",\"ZAF\",\"SOUTH AFRICA\",\"AIR\",\"\",\"\",\"\",\"\",\"\",\"\",\"202504\",\"TZA\",\"105441444\",\"\"\n",
      "\"zdzj\",\"1ab06a852dc44c9d\",\"I\",\"D\",\"O\",\"\",\"2025-04-24\",\"LUX\",\"LUXEMBOURG\",\"TZA\",\"TANZANIA\",\"WATERCOM (T) LIMITED\",\"\",\"\",\"HUSKY INJ MOLDING SYST SA\",\"\",\"\",\"850720000000\",\"Other lead-acid accumulators\",\"UPS 1100 7AHLEAD BATTERY MOD\",\"405.56524136092304\",\"\",\"11.0\",\"11.0\",\"1.0\",\"NMB\",\"\",\"0\",\"0\",\"0\",\"\",\"\",\"LUX\",\"LUXEMBOURG\",\"AIR\",\"\",\"\",\"\",\"\",\"\",\"\",\"202504\",\"TZA\",\"119864003\",\"\"\n",
      "\"zdzj\",\"757500b38d9d472b\",\"I\",\"D\",\"O\",\"\",\"2025-04-24\",\"JPN\",\"JAPAN\",\"ZWE\",\"ZIMBABWE\",\"LOVENESS DZIRUNI \",\"\",\"\",\"BE FORWADR CO.LTD\",\"\",\"\",\"870321900000\",\"Other:Motor cars and other motor vehicles p:Of a cylinder capacity>1,000cc but<=\",\"USED TOYOTA VITZ \",\"3963.850449282395\",\"\",\"970.0\",\"970.0\",\"1.0\",\"NMB\",\"\",\"0\",\"0\",\"0\",\"\",\"\",\"JPN\",\"JAPAN\",\"SEA\",\"\",\"\",\"\",\"\",\"\",\"\",\"202504\",\"TZA\",\"\",\"\"\n",
      "\"zdzj\",\"d7aa2e8b44e9412f\",\"I\",\"D\",\"O\",\"\",\"2025-04-24\",\"CHN\",\"CHINA\",\"TZA\",\"TANZANIA\",\"ZEIN AND BAHER COMPANY LIMITED\",\"\",\"\",\"WENZHOU YIRUI TRADE CO.,LIMITED\",\"\",\"\",\"630900100000\",\"Worn items of clothing\",\"USED CLOTHES GRADE B\",\"31199.84906331994\",\"\",\"23680.0\",\"23680.0\",\"23680.0\",\"KGM\",\"\",\"0\",\"0\",\"0\",\"\",\"\",\"CHN\",\"CHINA\",\"SEA\",\"\",\"\",\"\",\"\",\"\",\"\",\"202504\",\"TZA\",\"174216711\",\"\"\n",
      "\"zdzj\",\"3ada89e82a04488a\",\"I\",\"D\",\"O\",\"\",\"2025-04-24\",\"CHN\",\"CHINA\",\"TZA\",\"TANZANIA\",\"ZEIN AND BAHER COMPANY LIMITED\",\"\",\"\",\"WENZHOU YIRUI TRADE CO.,LIMITED\",\"\",\"\",\"420219000000\",\"Other worn items\",\"USED BAGS GRADE B\",\"474.32200990856546\",\"\",\"360.0\",\"360.0\",\"360.0\",\"NMB\",\"\",\"0\",\"0\",\"0\",\"\",\"\",\"CHN\",\"CHINA\",\"SEA\",\"\",\"\",\"\",\"\",\"\",\"\",\"202504\",\"TZA\",\"174216711\",\"\"\n",
      "\"zdzj\",\"d698a0caf0374bb0\",\"I\",\"D\",\"O\",\"\",\"2025-04-24\",\"ARE\",\"UNITED ARAB EMIRATES\",\"TZA\",\"TANZANIA\",\"FREIGHT IN TIME  ( TANZANIA ) LIMITED\",\"\",\"\",\"DELL PRODUCT (DUBAI)\",\"\",\"\",\"847170000000\",\"Storage units\",\"STORAGE UNIT\",\"2789.5578367389467\",\"\",\"3.0\",\"3.0\",\"1.0\",\"NMB\",\"\",\"0\",\"0\",\"0\",\"\",\"\",\"ARE\",\"UNITED ARAB EMIRATES\",\"AIR\",\"\",\"\",\"\",\"\",\"\",\"\",\"202504\",\"TZA\",\"107009272\",\"\"\n",
      "\"zdzj\",\"be24a3e62a1a41a7\",\"I\",\"D\",\"O\",\"\",\"2025-04-24\",\"JPN\",\"JAPAN\",\"TZA\",\"TANZANIA\",\"EVANGELISTIC ASSEMBLIES OF GOD TANZANIA (EAGT)\",\"\",\"\",\"SBT CO., LTD\",\"\",\"\",\"870323990000\",\"Other: of cylinder capacity exceeding 1500cc but not exceeding 3,000cc.\",\"USED TOYOTA LAND CRUISER PRADO\",\"21422.681019236716\",\"\",\"2240.0\",\"2240.0\",\"1.0\",\"NMB\",\"\",\"0\",\"0\",\"0\",\"\",\"\",\"JPN\",\"JAPAN\",\"SEA\",\"\",\"\",\"\",\"\",\"\",\"\",\"202504\",\"TZA\",\"103947200\",\"\"\n",
      "\"zdzj\",\"a40790dce2db48cd\",\"I\",\"D\",\"O\",\"\",\"2025-04-24\",\"TUR\",\"Turkey\",\"TZA\",\"TANZANIA\",\"BAKHRESA FOOD PRODUCTS LIMITED\",\"\",\"\",\"KRONES LCS CENTRE EAST AFRICA LTD\",\"\",\"\",\"853890000000\",\"Other:Parts suitable for use solely or principally with the apparatus of heading\",\"SPARE PARTS FOR KRONES FILLING LINE\",\"1186.9183264695057\",\"\",\"1.4\",\"1.4\",\"1.4\",\"KGM\",\"\",\"0\",\"0\",\"0\",\"\",\"\",\"DEU\",\"GERMANY\",\"AIR\",\"\",\"\",\"\",\"\",\"\",\"\",\"202504\",\"TZA\",\"100181517\",\"\"\n"
     ]
    }
   ],
   "source": [
    "!head -10 dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202504-IMP-RAW.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "c5b2d883-8ab2-47c3-9929-2363f93d2582",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !head -10 dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202504-EXP.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92e98974-1639-4370-ac86-b0e6d3f262ca",
   "metadata": {},
   "source": [
    "# 9 货代公司"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "a21de388-5500-41e5-b2c2-55821c637698",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hd = pd.read_csv('物流货代词典_20250820.csv')\n",
    "df_hd['物流货代'] = df_hd['物流货代'].str.upper()\n",
    "# df_hd\n",
    "\n",
    "hd_list = list(df_hd['物流货代'].values)\n",
    "\n",
    "from_dir = Path('dataset/data_20250724/20254t6_csv_correct1')\n",
    "to_dir = Path('dataset/data_20250724/20254t6_csv_correct1')\n",
    "\n",
    "country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# country_list = ['坦桑尼亚']\n",
    "# country_list = ['智利']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "beb83877-ee23-4e28-a049-bbfdc5dc4c2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# comp_forwarderagent(country_list, hd_list, from_dir, to_dir, cutoff=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "c4c1ce92-1f78-43e3-a288-a721da683ef4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 06:56:39,470 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 06:56:39,471 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:56:39,471 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:57:03,569 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 06:57:03,573 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 06:57:10,092 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 06:57:10,093 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 06:57:10,821 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 06:57:10,823 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 06:57:11,411 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 06:57:11,412 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:57:14,858 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 06:57:14,876 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 06:57:31,113 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 06:57:31,118 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:57:32,062 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 06:57:32,062 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 06:57:32,064 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 06:57:32,064 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 06:57:33,039 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 06:57:33,076 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:58:05,481 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 06:58:05,509 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 06:58:28,465 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 06:58:28,529 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:58:54,063 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 06:58:54,065 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 06:59:12,611 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 06:59:12,622 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 06:59:40,562 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 06:59:40,618 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 06:59:57,809 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 06:59:57,859 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 07:00:04,180 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 07:00:04,196 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 07:01:23,034 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 07:01:23,039 - File Process - INFO - country list ['坦桑尼亚'] takes 284 seconds\n",
      "2025-09-10 07:01:23,072 - Data Goverance - INFO - 1\n",
      "2025-09-10 07:01:24,473 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 07:01:24,475 - File Process - INFO - country list ['智利'] takes 285 seconds\n",
      "2025-09-10 07:01:24,518 - Data Goverance - INFO - 1\n",
      "2025-09-10 07:01:32,331 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 07:01:32,333 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 07:02:25,249 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 07:02:25,251 - File Process - INFO - dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 07:03:19,678 - File Process - INFO - saved to : dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 07:03:19,679 - File Process - INFO - country list ['乌克兰'] takes 400 seconds\n",
      "2025-09-10 07:03:19,766 - Data Goverance - INFO - 1\n",
      "2025-09-10 07:03:19,771 - Data Goverance - INFO - all comp_forwarderagent takes 6.67171037197113 minutes\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "params_list = [([ctry], hd_list, from_dir, to_dir, 0.95) for ctry in country_list]\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: comp_forwarderagent(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        logger.info(future.result())\n",
    "        # pass\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'all comp_forwarderagent takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ea1e27b-dac3-4a1b-bba3-ba5ebfc2b937",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "254a5be4-dbf3-46b1-ae53-70cbe306ef7e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "30bc2e64-9f5a-45bd-b433-f62c3bd19212",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pth2 = Path('dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "76f0d428-5276-4022-8031-a88d564a365d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df2 = pd.read_csv(pth2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "6ae11d67-2646-4d81-b6d6-ef42355cd357",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# df2['importer_forwarderagent'] = df2['importername'].str.upper().apply(lambda x: mapping_forwarderagent(x, choices, cutoff=0.9))\n",
    "# df2['supplier_forwarderagent'] = df2['suppliername'].str.upper().apply(lambda x: mapping_forwarderagent(x, choices, cutoff=0.9))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "cfaa6715-cda5-45b5-8a6b-29e769b43d50",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set(list(df2['importer_forwarderagent'].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "8b97d0da-7215-42ef-91f1-973307692cd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set(list(df2['supplier_forwarderagent'].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "f5640a84-81e0-430a-bf49-008142113d8b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# df2.loc[df2['importer_forwarderagent']!=0, ['dataid', 'importername', 'importer_forwarderagent']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea2431fc-9c32-4ada-9ce6-d377bd64b1f1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3e5294ce-9f93-48ed-85f7-197a6c1ab582",
   "metadata": {},
   "source": [
    "# 9 空值率与指标统计，并更新数字"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26f913de-8799-42ed-b6fb-178a6a5263ba",
   "metadata": {},
   "source": [
    "## 9.1 总和指标计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cea3c8e7-839f-4afc-a348-e75775489517",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# # country_list = ['坦桑尼亚']\n",
    "# country_list = ['乌克兰']\n",
    "# csv_files_path = Path('dataset/data_20250724/20251t3_csv_final_tmp/files11')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "322d792b-eb24-47c3-99e0-ce9a24602ed1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# start = time.time()\n",
    "\n",
    "# params_list = [([ctry], csv_files_path, country36_dict) for ctry in country_list]\n",
    "# results = []\n",
    "# with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "#     # 提交多个任务\n",
    "#     futures = [\n",
    "#         executor.submit(lambda p: compute_stat4(*p), params)\n",
    "#         for params in params_list\n",
    "#     ]\n",
    "#     # 按完成顺序获取结果\n",
    "#     for future in as_completed(futures):\n",
    "#         res = future.result()\n",
    "#         logger.info(res.shape)\n",
    "#         results.append(res)\n",
    "\n",
    "# stat_rslt1 = pd.concat([x for x in results], axis=0, ignore_index=True)\n",
    "\n",
    "# end = time.time()\n",
    "# logger.info(f'get_type_enumvalue takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "13d2681a-6c8b-4c90-a2c2-af0d9e3f4344",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 单线程\n",
    "# stat_rslt1 = compute_stat4(country_list, csv_files_path, country36_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "69fe5b33-ff94-4ab8-89f7-e1f0d8eba209",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # stat_rslt1.to_csv('result/statistic_result_sum_20250820.csv', sep=',', index=False, header=True, encoding='utf-8')\n",
    "# # stat_rslt1.to_csv('result/statistic_result_sum_20250908.csv', sep=',', index=False, header=True, encoding='utf-8')\n",
    "# stat_rslt1.to_csv('result/statistic_result_sum_20250917.csv', sep=',', index=False, header=True, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9b9fb37b-188d-4d9e-b104-7fe6c7924830",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# stat_rslt1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "761d9e88-ff0f-4c68-9256-e5dc0ba8db00",
   "metadata": {},
   "outputs": [],
   "source": [
    "# future.result().shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c699184-2580-4723-81b7-b380d2fb8460",
   "metadata": {},
   "source": [
    "## 9.2 空值率指标计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "15b35f4a-f893-43b7-82a2-4799092cf839",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # country_list = ['智利', '坦桑尼亚', '乌克兰']\n",
    "# # country_list = ['坦桑尼亚']\n",
    "# country_list = ['乌克兰']\n",
    "# # csv_files_path = Path('dataset/data_20250724/20251t3_csv_final/files')\n",
    "# # csv_files_path = Path('dataset/data_20250724/20251t3_csv_final_tmp/final_files')\n",
    "# csv_files_path = Path('dataset/data_20250724/20251t3_csv_final_tmp/files11')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ad3ecb31-8190-49e5-9c60-865eda31a9c9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# stat_rslt2 = compute_stat5(country_list, csv_files_path, country36_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dee180d9-2862-4db4-b03f-1b129bee5632",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# start = time.time()\n",
    "# params_list = [([ctry], csv_files_path, country36_dict) for ctry in country_list]\n",
    "\n",
    "# results = []\n",
    "# with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "#     # 提交多个任务\n",
    "#     futures = [\n",
    "#         executor.submit(lambda p: compute_stat5(*p), params)\n",
    "#         for params in params_list\n",
    "#     ]\n",
    "#     # 按完成顺序获取结果\n",
    "#     for future in as_completed(futures):\n",
    "#         res = future.result()\n",
    "#         logger.info(res.shape)\n",
    "#         results.append(res)\n",
    "\n",
    "# stat_rslt2 = pd.concat([x for x in results], axis=0, ignore_index=True)\n",
    "# # stat_rslt2\n",
    "\n",
    "# end = time.time()\n",
    "# logger.info(f'get_type_enumvalue takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3549e3f8-0421-4ff9-a1dd-0f78589e87be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # stat_rslt2.to_csv('result/statistic_result_rus_20250820.csv', sep=',', index=False, header=True, encoding='utf-8')\n",
    "# # stat_rslt2.to_csv('result/statistic_result_rus_20250908.csv', sep=',', index=False, header=True, encoding='utf-8')\n",
    "# stat_rslt2.to_csv('result/statistic_result_rus_20250917.csv', sep=',', index=False, header=True, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "384e3d14-9211-41f7-a23c-1024c00e8792",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# stat_rslt2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ca44b1d4-dd32-4216-8b92-59e093c9bdea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wc -l dataset/data_20250724/20251t3_csv_final_tmp/files11/乌克兰/UKR_202502_EXP.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52802231-a7f5-4877-9278-199939970fd9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d342a52c-d4e7-491b-bbd7-82ebc1df4d32",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d598c13-4a73-4ce4-8e15-a56aeca99cd9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "04b3395e-9c1c-4648-8b1b-fe38473c1ad8",
   "metadata": {},
   "source": [
    "# 10 获取HS码映射关系"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cd501e74-e687-4031-9165-94b50edcd1f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from_path = Path('dataset/data_20250724/20251t3_csv_correct1')\n",
    "# to_path = Path('result/20251t3_hs_mapping')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "52d02c0b-b4c9-4bc1-ae2a-0f807ad89cde",
   "metadata": {},
   "outputs": [],
   "source": [
    "# country_list = ['印度', '英国', '越南', '阿根廷', '埃塞俄比亚', '巴基斯坦', '俄罗斯陆运', '厄瓜多尔'\n",
    "#                 , '菲律宾', '哥伦比亚', '哥斯达黎加', '哈萨克斯坦', '加纳', '喀麦隆', '科特迪瓦', '肯尼亚'\n",
    "#                 , '莱索托', '马拉维', '美国', '孟加拉', '秘鲁', '秘鲁-海运', '秘鲁-空运', '墨西哥'\n",
    "#                 , '纳米比亚', '尼日利亚', '斯里兰卡', '坦桑尼亚', '乌干达', '乌克兰', '乌拉圭', '乌兹别克斯坦'\n",
    "#                 , '智利', '巴拉圭', '巴拿马', '博兹瓦纳']\n",
    "# # country_list = ['尼日利亚', '博兹瓦纳']\n",
    "# # country_list = ['阿根廷']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "51a075b0-d2e9-4b79-a93b-e45fc41e9a52",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# get_hs_mapping(from_path, to_path, country_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d060e876-48fd-4612-9dca-3fdc2d228b07",
   "metadata": {},
   "outputs": [],
   "source": [
    "# start = time.time()\n",
    "# params_list = [(from_path, to_path, [ctry]) for ctry in country_list]\n",
    "\n",
    "# results = []\n",
    "# with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "#     # 提交多个任务\n",
    "#     futures = [\n",
    "#         executor.submit(lambda p: get_hs_mapping(*p), params)\n",
    "#         for params in params_list\n",
    "#     ]\n",
    "#     # 按完成顺序获取结果\n",
    "#     for future in as_completed(futures):\n",
    "#         res = future.result()\n",
    "#         logger.info(res.shape)\n",
    "#         results.append(res)\n",
    "\n",
    "# end = time.time()\n",
    "# logger.info(f'get_type_enumvalue takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ab74dae3-3681-43a3-9e01-310981f9c1a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ab1d982-3dd8-4f50-bcb9-e8b4ef73027c",
   "metadata": {},
   "source": [
    "# 11 获取成交方式、运输方式、付款方式、"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "be3db185-dbf5-4568-aa60-8b1127378e00",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transportterm    运输方式\n",
    "# tradeterm    成交方式\n",
    "# paymentterm    付款方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "970730e1-92bc-40b1-9b4f-2d17552668af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from_path = Path('dataset/data_20250724/20251t3_csv_correct1')\n",
    "# to_path = Path('result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "3e0843a1-855d-4358-bebe-4eb662be9a17",
   "metadata": {},
   "outputs": [],
   "source": [
    "# country_list = ['印度', '英国', '越南', '阿根廷', '埃塞俄比亚', '巴基斯坦', '俄罗斯陆运', '厄瓜多尔'\n",
    "#                 , '菲律宾', '哥伦比亚', '哥斯达黎加', '哈萨克斯坦', '加纳', '喀麦隆', '科特迪瓦', '肯尼亚'\n",
    "#                 , '莱索托', '马拉维', '美国', '孟加拉', '秘鲁', '秘鲁-海运', '秘鲁-空运', '墨西哥'\n",
    "#                 , '纳米比亚', '尼日利亚', '斯里兰卡', '坦桑尼亚', '乌干达', '乌克兰', '乌拉圭', '乌兹别克斯坦'\n",
    "#                 , '智利', '巴拉圭', '巴拿马', '博兹瓦纳']\n",
    "# # country_list = ['阿根廷']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "ed8b9225-2af5-4a0a-826e-ae6495f086ac",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# rslt = get_type_enumvalue(from_path, to_path, country_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "2847fd74-377e-4a6c-be00-14448f1ed937",
   "metadata": {},
   "outputs": [],
   "source": [
    "# start = time.time()\n",
    "# params_list = [(from_path, to_path, [ctry]) for ctry in country_list]\n",
    "\n",
    "# results = []\n",
    "# with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "#     # 提交多个任务\n",
    "#     futures = [\n",
    "#         executor.submit(lambda p: get_type_enumvalue(*p), params)\n",
    "#         for params in params_list\n",
    "#     ]\n",
    "#     # 按完成顺序获取结果\n",
    "#     for future in as_completed(futures):\n",
    "#         res = future.result()\n",
    "#         logger.info(res.shape)\n",
    "#         results.append(res)\n",
    "\n",
    "# # rslt = pd.concat([future.result() for future in futures], axis=0, ignore_index=True)\n",
    "# end = time.time()\n",
    "# logger.info(f'get_type_enumvalue takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "7739976a-4002-4170-bee3-3f2050dc52d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "49bec0f3-f253-4585-9e3b-59e34ec76b8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# rslt.to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "5dac3146-6b0c-4137-af4c-bbbc46fa2702",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# rslt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b78e1669-6305-4fd5-8368-f558bea09d78",
   "metadata": {},
   "source": [
    "# 12 上传的csv文件格式处理"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6471fa36-2d86-4ae8-9dda-a14a686d71cd",
   "metadata": {},
   "source": [
    "## 12.1 去掉\"\\\\N\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "3a8ac44e-faec-40ba-aa71-1f8da8826088",
   "metadata": {},
   "outputs": [],
   "source": [
    "# country_list = ['印度', '英国', '越南', '阿根廷', '埃塞俄比亚', '巴基斯坦', '俄罗斯陆运', '厄瓜多尔'\n",
    "#                 , '菲律宾', '哥伦比亚', '哥斯达黎加', '哈萨克斯坦', '加纳', '喀麦隆', '科特迪瓦', '肯尼亚'\n",
    "#                 , '莱索托', '马拉维', '美国', '孟加拉', '秘鲁', '秘鲁-海运', '秘鲁-空运', '墨西哥'\n",
    "#                 , '纳米比亚', '尼日利亚', '斯里兰卡', '坦桑尼亚', '乌干达', '乌克兰', '乌拉圭', '乌兹别克斯坦'\n",
    "#                 , '智利', '巴拉圭', '巴拿马', '博兹瓦纳']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "e79f6b8d-369c-4949-a1c3-f4f1c206e87e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 获取文件目录，区分出口、进口数据\n",
    "# from_path = Path('dataset/data_20250724/20251t3_csv_final_tmp/final_files')\n",
    "\n",
    "# # file_tree = distinct_ie_data2(from_path, country_list)\n",
    "# to_path = Path('dataset/data_20250724/20251t3_csv_final/final_files')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "39872c1b-733e-4e75-9e5f-bff06b031204",
   "metadata": {},
   "outputs": [],
   "source": [
    "# country_list = ['俄罗斯陆运']\n",
    "\n",
    "# # 获取文件目录，区分出口、进口数据\n",
    "# from_path = Path('dataset/data_20250724/20251t3_csv_final_tmp/RUS')\n",
    "\n",
    "# # file_tree = distinct_ie_data2(from_path, country_list)\n",
    "# to_path = Path('dataset/data_20250724/20251t3_csv_final/final_files')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "645ba27e-01cc-410f-bda5-0a12baa2ef14",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# start = time.time()\n",
    "# params_list = [([ctry], from_path, to_path, country36_dict) for ctry in country_list]\n",
    "\n",
    "# with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "#     # 提交多个任务\n",
    "#     futures = [\n",
    "#         executor.submit(lambda p: replace_tab_to_norm(*p), params)\n",
    "#         for params in params_list\n",
    "#     ]\n",
    "#     # 按完成顺序获取结果\n",
    "#     for future in as_completed(futures):\n",
    "#         # logger.info(future.result().shape)\n",
    "#         pass\n",
    "\n",
    "\n",
    "# end = time.time()\n",
    "# logger.info(f'replace_tab_to_norm takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "e67ef03c-0b6b-45c4-a6ac-bf9c59dc3531",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# params_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "bc5deb04-4284-4bed-ad2c-e7c587fd62d2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# replace_tab_to_norm(country_list, from_path, to_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a2bd569-e749-401e-ae3b-bc023b57d404",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "aecbb604-0068-488b-9bc6-e27139355b9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !head -3 dataset/data_20250724/20251t3_csv_final/files/ARG_202501_EXP.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e2b1227-9fe0-4681-94f1-a9250b5b5418",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c938f96b-ebd7-4e93-a274-0d860fc43c99",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "dc805f0f-e55c-49e6-8b9f-f9a91772308b",
   "metadata": {},
   "source": [
    "# 13 校验与更正"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "f05bda2e-3ec0-433a-8143-878b7f93b56e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# col_type = get_col_str_type(config.target_cols[0:-2])\n",
    "# # tab_tmp = pd.read_csv('dataset/data_20250724/UZB_202501_IMP.csv',encoding='utf-8',header=0,dtype=col_type)\n",
    "# column_names = config.target_cols[0:-2]\n",
    "# # dataset/data_20250724/20251t3_csv_final_tmp/final_files/BGD_202501_EXP.csv\n",
    "# # dataset/data_20250724/20251t3_csv_final_tmp/final_files/BWA_202503_EXP.csv\n",
    "\n",
    "# tab_tmp = pd.read_csv('dataset/data_20250724/20251t3_csv_final_tmp/final_files/BWA_202503_EXP.csv' \n",
    "#                       , header=None, names=column_names\n",
    "#                       , sep=','\n",
    "#                       , quotechar='\"'\n",
    "#                       , lineterminator='\\n'\n",
    "#                       , on_bad_lines='skip'\n",
    "#                       , encoding='utf-8',dtype=col_type)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d9dac8e-026c-44ac-877b-d08da05d2cd4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8c7a3f9-0294-499a-9c41-82348700b305",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bb17094-94a5-4e71-adc2-621d7b135de7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "580353fc-fb87-4a1f-9f50-13e210dbe556",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2560e7a6-8f6e-4a96-978d-dea704194a2f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "25a80fff-3bd3-44f0-be12-b36efa246b6a",
   "metadata": {},
   "source": [
    "# end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "893235b3-693f-47df-bad1-6256b394e865",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 07:06:32,450 - Data Goverance - INFO - 18.75146779616674 minutes\n",
      "2025-09-10 07:06:32,451 - Data Goverance - INFO - All done!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "all_end=time.time()\n",
    "logger.info(f'{(all_end-all_start)/60} minutes')\n",
    "logger.info('All done!\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "409ea3bf-dca2-4bd1-b59d-ef2045e32deb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# str_type = dict()\n",
    "# for col in config.target_cols:\n",
    "#     str_type[col] = 'str'\n",
    "# str_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c309bd2d-0ddb-4aeb-a788-672b2daea563",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "763f4c3c-4b7e-4f67-85ba-450a11443419",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
